[Bug]: test_nhop_group_interface_flap fails

[AntonHryshchuk]

Issue Description

Test failed with code from master/202411, but passed with 202405 code
Same image, topo(t1-lag-c224o8), and SKU(Mellanox-SN5600-C224O8) on the switch.

Results you see

The traceback from failed test:

 def test_nhop_group_interface_flap(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, ptfadapter,
                                       gather_facts, enum_rand_one_frontend_asic_index, fanouthosts):
        """
        Test for packet drop when route is added with ECMP and all ECMP member's
        interfaces are down. Use kernel flag 'arp_evict_nocarrier' to disable ARP
        eviction from the kernel when the interface goes down.
        Kernel flag is used to easily recreate the scenario of ECMP with no
        Nexthop members. Without this kernel flag, static route addition fails when
        Nexthop ARP entries are not resolved.
        """
        duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
        asic = duthost.asic_instance(enum_rand_one_frontend_asic_index)
    
        # Check Gather facts IP Interface is active one
        ip_ifaces = asic.get_active_ip_interfaces(tbinfo).keys()
        pytest_assert(len(ip_ifaces), "No IP interfaces found")
        pytest_assert(gather_facts['src_router_intf_name'] in ip_ifaces, "Selected IP interfaces is not active")
    
        # Generate ARP entries
        arp_count = 2
        logger.debug("ARP interface: %s", gather_facts['src_router_intf_name'])
        for i in range(0, len(gather_facts['src_port'])):
            logger.debug("Src port: %s, src port index: %d", gather_facts['src_port'][i], gather_facts['src_port_ids'][i])
    
        arplist = Arp(duthost, asic, arp_count, gather_facts['src_router_intf_name'])
        neighbor_mac = [neighbor[1].lower() for neighbor in arplist.ip_mac_list]
        ip_route = "192.168.100.50"
        ip_prefix = ip_route + "/32"
        ip_ttl = 64
    
        arp_noevict_cmd = "echo 0 > /proc/sys/net/ipv4/conf/%s/arp_evict_nocarrier"
        arp_evict_cmd = "echo 1 > /proc/sys/net/ipv4/conf/%s/arp_evict_nocarrier"
    
        # create nexthop group
        nhop = IPRoutes(duthost, asic)
    
        try:
            rtr_mac = asic.get_router_mac()
            arplist.arps_add()
            ips = [arplist.ip_mac_list[x].ip for x in range(arp_count)]
            # add IP route
            nhop.ip_nhops = []
    
            # Enable kernel flag to not evict ARP entries when the interface goes down
            # and shut the fanout switch ports.
            asic.command(arp_noevict_cmd % gather_facts['src_router_intf_name'])
            for i in range(0, len(gather_facts['src_port'])):
                fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname,
                                                                gather_facts['src_port'][i])
                logger.debug("Shut fanout sw: %s, port: %s", fanout, fanout_port)
                if is_vs_device(duthost) is False:
                    fanout.no_shutdown(fanout_port)
            nhop.add_ip_route(ip_prefix, ips)
    
            nhop.program_routes()
            # wait for routes to be synced and programmed
            pkt, exp_pkt = build_pkt(rtr_mac, ip_route, ip_ttl, 1)
            pkt_count = 1
    
            logger.debug("Sending packet on %s", gather_facts['dst_port'][0])
            testutils.send(ptfadapter, gather_facts['dst_port_ids'][0], pkt, pkt_count)
>           testutils.verify_no_packet_any(ptfadapter, exp_pkt, ports=gather_facts['src_port_ids'])

arp_count  = 2
arp_evict_cmd = 'echo 1 > /proc/sys/net/ipv4/conf/%s/arp_evict_nocarrier'
arp_noevict_cmd = 'echo 0 > /proc/sys/net/ipv4/conf/%s/arp_evict_nocarrier'
arplist    = <test_nhop_group.Arp object at 0x7fc22ee279a0>
asic       = <SonicAsic 0>
duthost    = <MultiAsicSonicHost mtvr-moose-13>
duthosts   = [<MultiAsicSonicHost mtvr-moose-13>]
enum_rand_one_frontend_asic_index = None
enum_rand_one_per_hwsku_frontend_hostname = 'mtvr-moose-13'
exp_pkt    = <ptf.mask.Mask object at 0x7fc22ee37670>
fanout     = { os: 'sonic', hostname: 'mtvr-moose-19', device_type: 'FanoutLeaf' }
fanout_port = 'Ethernet1'
fanouthosts = {'mtvr-moose-19': { os: 'sonic', hostname: 'mtvr-moose-19', device_type: 'FanoutLeaf' }}
gather_facts = {'dst_host_ipv4': '10.0.0.5', 'dst_host_ipv6': 'fc00::a', 'dst_host_mac': '82:c3:02:91:c3:13', 'dst_port': ['Ethernet2', 'Ethernet3'], ...}
i          = 1
ip_ifaces  = dict_keys(['Ethernet64', 'Ethernet65', 'Ethernet66', 'Ethernet67', 'Ethernet80', 'Ethernet81', 'Ethernet82', 'Ethernet...105', 'PortChannel108', 'PortChannel1011', 'PortChannel1014', 'PortChannel1017', 'PortChannel1020', 'PortChannel1023'])
ip_prefix  = '192.168.100.50/32'
ip_route   = '192.168.100.50'
ip_ttl     = 64
ips        = ['172.16.0.11', '172.16.0.12']
neighbor_mac = ['c0:ff:ee:00:00:0b', 'c0:ff:ee:00:00:0c']
nhop       = <test_nhop_group.IPRoutes object at 0x7fc22ebd7850>
pkt        = <Ether  dst=b0:cf:0e:b3:20:00 src=00:11:22:33:44:55 type=IPv4 |<IP  ihl=None tos=0x0 id=1 frag=0 ttl=64 proto=tcp src=...st=192.168.100.50 |<TCP  sport=101 dport=201 flags=S |<Raw  load='test_nhop_group test_nhop_group test_nhop_grou' |>>>>
pkt_count  = 1
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
rtr_mac    = 'b0:cf:0e:b3:20:00'
tbinfo     = {'comment': 'Mellanox MTR testbed', 'conf-name': 'mtvr-moose-13-t1-lag-c224o8', 'duts': ['mtvr-moose-13'], 'duts_map': {'mtvr-moose-13': 0}, ...}

ipfwd/test_nhop_group.py:898: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/local/lib/python3.8/dist-packages/ptf/testutils.py:3346: in verify_no_packet_any
    verify_no_packet(test, pkt, (device, port), timeout=timeout)
        device     = 0
        device_number = 0
        pkt        = <ptf.mask.Mask object at 0x7fc22ee37670>
        port       = 1
        ports      = [0, 1]
        test       = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
        timeout    = None
/usr/local/lib/python3.8/dist-packages/ptf/testutils.py:3270: in verify_no_packet
    test.fail(
        device     = 0
        pkt        = <ptf.mask.Mask object at 0x7fc22ee37670>
        port       = 1
        port_id    = (0, 1)
        result     = PollSuccess(device=0, port=1, packet=b'\xc0\xff\xee\x00\x00\x0c\xb0\xcf\x0e\xb3 \x00\x08\x00E\x00\x00V\x00\x01\x00\x00...00\x00\x00\x00\x00\x00P\x02 \x00\x91\n\x00\x00test_nhop_group test_nhop_group test_nhop_grou', time=1735032321.9215798)
        test       = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
        timeout    = 0.1
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
msg = 'Received packet that we expected not to receive on device 0, port 1.\n========== RECEIVED ==========\n0000  C0 FF EE ...70 5F  group test_nhop_\n0060  67 72 6F 75                                      grou\n==============================\n'

    def fail(self, msg=None):
        """Fail immediately, with the given message."""
>       raise self.failureException(msg)
E       AssertionError: Received packet that we expected not to receive on device 0, port 1.
E       ========== RECEIVED ==========
E       0000  C0 FF EE 00 00 0C B0 CF 0E B3 20 00 08 00 45 00  .......... ...E.
E       0010  00 56 00 01 00 00 3F 06 43 63 13 00 00 64 C0 A8  .V....?.Cc...d..
E       0020  64 32 00 65 00 C9 00 00 00 00 00 00 00 00 50 02  d2.e..........P.
E       0030  20 00 91 0A 00 00 74 65 73 74 5F 6E 68 6F 70 5F   .....test_nhop_
E       0040  67 72 6F 75 70 20 74 65 73 74 5F 6E 68 6F 70 5F  group test_nhop_
E       0050  67 72 6F 75 70 20 74 65 73 74 5F 6E 68 6F 70 5F  group test_nhop_
E       0060  67 72 6F 75                                      grou
E       ==============================

msg        = 'Received packet that we expected not to receive on device 0, port 1.\n========== RECEIVED ==========\n0000  C0 FF EE ...70 5F  group test_nhop_\n0060  67 72 6F 75                                      grou\n==============================\n'
self       = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>

Results you expected to see

test passed

Is it platform specific

generic

Relevant log output

No response

Output of show version

No response

Attach files (if any)

No response