Merge branch 'master' into eni_cntrs

.azure-pipelines/build_and_install_module.sh

@@ -26,62 +26,60 @@ function build_and_install_kmodule()
     SUBLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f3)
 
     # Install the required debian packages to build the kernel modules
+    apt-get update
     apt-get install -y build-essential linux-headers-${KERNEL_RELEASE} autoconf pkg-config fakeroot
-    apt-get install -y flex bison libssl-dev libelf-dev
+    apt-get install -y flex bison libssl-dev libelf-dev dwarves
     apt-get install -y libnl-route-3-200 libnl-route-3-dev libnl-cli-3-200 libnl-cli-3-dev libnl-3-dev
 
     # Add the apt source mirrors and download the linux image source code
     cp /etc/apt/sources.list /etc/apt/sources.list.bk
     sed -i "s/^# deb-src/deb-src/g" /etc/apt/sources.list
     apt-get update
-    apt-get source linux-image-unsigned-$(uname -r) > source.log
+    KERNEL_PACKAGE_SOURCE=$(apt-cache show linux-image-unsigned-${KERNEL_RELEASE} | grep ^Source: | cut -d':' -f 2)
+    KERNEL_PACKAGE_VERSION=$(apt-cache show linux-image-unsigned-${KERNEL_RELEASE} | grep ^Version: | cut -d':' -f 2)
+    SOURCE_PACKAGE_VERSION=$(apt-cache showsrc ${KERNEL_PACKAGE_SOURCE} | grep ^Version: | cut -d':' -f 2)
+    if [ ${KERNEL_PACKAGE_VERSION} != ${SOURCE_PACKAGE_VERSION} ]; then
+        echo "WARNING: the running kernel version (${KERNEL_PACKAGE_VERSION}) doesn't match the source package " \
+            "version (${SOURCE_PACKAGE_VERSION}) being downloaded. There's no guarantee the module being downloaded " \
+            "can be loaded into the kernel or function correctly. If possible, please update your kernel and reboot " \
+            "your system so that it's running the matching kernel version." >&2
+        echo "Continuing with the build anyways" >&2
+    fi
+    apt-get source linux-image-unsigned-${KERNEL_RELEASE} > source.log
 
     # Recover the original apt sources list
     cp /etc/apt/sources.list.bk /etc/apt/sources.list
     apt-get update
 
     # Build the Linux kernel module drivers/net/team and vrf
     cd $(find . -maxdepth 1 -type d | grep -v "^.$")
+    if [ -e debian/debian.env ]; then
+        source debian/debian.env
+        if [ -n "${DEBIAN}" -a -e ${DEBIAN}/reconstruct ]; then
+            bash ${DEBIAN}/reconstruct
+        fi
+    fi
     make  allmodconfig
     mv .config .config.bk
     cp /boot/config-$(uname -r) .config
     grep NET_TEAM .config.bk >> .config
-    echo CONFIG_NET_VRF=m >> .config
-    echo CONFIG_MACSEC=m >> .config
-    echo CONFIG_NET_VENDOR_MICROSOFT=y >> .config
-    echo CONFIG_MICROSOFT_MANA=m >> .config
-    echo CONFIG_SYSTEM_REVOCATION_LIST=n >> .config
     make VERSION=$VERSION PATCHLEVEL=$PATCHLEVEL SUBLEVEL=$SUBLEVEL EXTRAVERSION=-${EXTRAVERSION} LOCALVERSION=-${LOCALVERSION} modules_prepare
-    make M=drivers/net/team
+    cp /usr/src/linux-headers-$(uname -r)/Module.symvers .
+    make -j$(nproc) M=drivers/net/team
     mv drivers/net/Makefile drivers/net/Makefile.bak
     echo 'obj-$(CONFIG_NET_VRF) += vrf.o' > drivers/net/Makefile
     echo 'obj-$(CONFIG_MACSEC) += macsec.o' >> drivers/net/Makefile
-    make M=drivers/net
+    make -j$(nproc) M=drivers/net
 
     # Install the module
-    TEAM_DIR=$(echo /lib/modules/$(uname -r)/kernel/net/team)
-    NET_DIR=$(echo /lib/modules/$(uname -r)/kernel/net)
-    if [ ! -e "$TEAM_DIR/team.ko" ]; then
-        mkdir -p $TEAM_DIR
-        cp drivers/net/team/*.ko $TEAM_DIR/
-        modinfo $TEAM_DIR/team.ko
-        depmod
-        modprobe team
-    fi
-    if [ ! -e "$NET_DIR/vrf.ko" ]; then
-        mkdir -p $NET_DIR
-        cp drivers/net/vrf.ko $NET_DIR/
-        modinfo $NET_DIR/vrf.ko
-        depmod
-        modprobe vrf
-    fi
-    if [ ! -e "$NET_DIR/macsec.ko" ]; then
-        mkdir -p $NET_DIR
-        cp drivers/net/macsec.ko $NET_DIR/
-        modinfo $NET_DIR/macsec.ko
-        depmod
-        modprobe macsec
-    fi
+    SONIC_MODULES_DIR=/lib/modules/$(uname -r)/updates/sonic
+    mkdir -p $SONIC_MODULES_DIR
+    cp drivers/net/team/*.ko drivers/net/vrf.ko drivers/net/macsec.ko $SONIC_MODULES_DIR/
+    depmod
+    modinfo team vrf macsec
+    modprobe team
+    modprobe vrf
+    modprobe macsec
 
     cd /tmp
     rm -rf $WORKDIR

cfgmgr/buffer_pool_mellanox.lua

@@ -417,10 +417,12 @@ local pool_size
 if shp_size then
     accumulative_occupied_buffer = accumulative_occupied_buffer + shp_size
 end
+
+local available_buffer = mmu_size - accumulative_occupied_buffer
 if ingress_pool_count == 1 then
-    pool_size = mmu_size - accumulative_occupied_buffer
+    pool_size = available_buffer
 else
-    pool_size = (mmu_size - accumulative_occupied_buffer) / 2
+    pool_size = available_buffer / 2
 end
 
 if pool_size > ceiling_mmu_size then
@@ -429,12 +431,19 @@ end
 
 local shp_deployed = false
 for i = 1, #pools_need_update, 1 do
+    local percentage = tonumber(redis.call('HGET', pools_need_update[i], 'percentage'))
+    local effective_pool_size
+    if percentage ~= nil and percentage >= 0 then
+        effective_pool_size = available_buffer * percentage / 100
+    else
+        effective_pool_size = pool_size
+    end
     local pool_name = string.match(pools_need_update[i], "BUFFER_POOL|([^%s]+)$")
     if shp_size ~= 0 and pool_name == "ingress_lossless_pool" then
-        table.insert(result, pool_name .. ":" .. math.ceil(pool_size) .. ":" .. math.ceil(shp_size))
+        table.insert(result, pool_name .. ":" .. math.ceil(effective_pool_size) .. ":" .. math.ceil(shp_size))
         shp_deployed = true
     else
-        table.insert(result, pool_name .. ":" .. math.ceil(pool_size))
+        table.insert(result, pool_name .. ":" .. math.ceil(effective_pool_size))
     end
 end
 

cfgmgr/buffermgrd.cpp

@@ -46,7 +46,7 @@ void dump_db_item(KeyOpFieldsValuesTuple &db_item)
 
 void write_to_state_db(shared_ptr<vector<KeyOpFieldsValuesTuple>> db_items_ptr)
 {
-    DBConnector db("STATE_DB", 0, true);
+    DBConnector db("STATE_DB", 0);
     auto &db_items = *db_items_ptr;
     for (auto &db_item : db_items)
     {

cfgmgr/vlanmgr.cpp

@@ -95,35 +95,13 @@ VlanMgr::VlanMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, c
     std::string res;
     EXEC_WITH_ERROR_THROW(cmds, res);
 
-    // The generated command is:
-    // /bin/echo 1 > /sys/class/net/Bridge/bridge/vlan_filtering
-    const std::string echo_cmd = std::string("")
-      + ECHO_CMD + " 1 > /sys/class/net/" + DOT1Q_BRIDGE_NAME + "/bridge/vlan_filtering";
-
-    int ret = swss::exec(echo_cmd, res);
-    /* echo will fail in virtual switch since /sys directory is read-only.
-     * need to use ip command to setup the vlan_filtering which is not available in debian 8.
-     * Once we move sonic to debian 9, we can use IP command by default
-     * ip command available in Debian 9 to create a bridge with a vlan filtering:
-     * /sbin/ip link add Bridge up type bridge vlan_filtering 1 */
-    if (ret != 0)
-    {
-        const std::string echo_cmd_backup = std::string("")
-          + IP_CMD + " link set " + DOT1Q_BRIDGE_NAME + " type bridge vlan_filtering 1";
-
-        EXEC_WITH_ERROR_THROW(echo_cmd_backup, res);
-    }
-
-    // not learn from link-local frames
-    // /bin/echo 1 > /sys/class/net/Bridge/bridge/no_linklocal_learn
-    const std::string no_ll_learn_cmd = std::string("")
-      + ECHO_CMD + " 1 > /sys/class/net/" + DOT1Q_BRIDGE_NAME + "/bridge/no_linklocal_learn";
-
-    ret = swss::exec(no_ll_learn_cmd, res);
-    if (ret != 0) {
-        EXEC_WITH_ERROR_THROW(no_ll_learn_cmd, res);
-    }
+    // /sbin/ip link set Bridge type bridge vlan_filtering 1
+    const std::string vlan_filtering_cmd = std::string(IP_CMD) + " link set " + DOT1Q_BRIDGE_NAME + " type bridge vlan_filtering 1";
+    EXEC_WITH_ERROR_THROW(vlan_filtering_cmd, res);
 
+    // /sbin/ip link set Bridge type bridge no_linklocal_learn 1
+    const std::string no_ll_learn_cmd = std::string(IP_CMD) + " link set " + DOT1Q_BRIDGE_NAME + " type bridge no_linklocal_learn 1";
+    EXEC_WITH_ERROR_THROW(no_ll_learn_cmd, res);
 }
 
 bool VlanMgr::addHostVlan(int vlan_id)

cfgmgr/vxlanmgr.cpp

@@ -139,6 +139,15 @@ static int cmdDeleteVxlan(const swss::VxlanMgr::VxlanInfo & info, std::string &
     return swss::exec(cmd.str(), res);
 }
 
+static int cmdVxlanLearningOff(const swss::VxlanMgr::VxlanInfo & info, std::string & res)
+{
+    // bridge link set dev {{VXLAN}} learning off
+    ostringstream cmd;
+    cmd << BRIDGE_CMD << " link set dev "
+        << shellquote(info.m_vxlan) << " learning off";
+    return swss::exec(cmd.str(), res);
+}
+
 static int cmdDeleteVxlanFromVxlanIf(const swss::VxlanMgr::VxlanInfo & info, std::string & res)
 {
     // brctl delif {{VXLAN_IF}} {{VXLAN}}
@@ -683,6 +692,7 @@ bool VxlanMgr::doVxlanEvpnNvoCreateTask(const KeyOpFieldsValuesTuple & t)
         }
         if (field == SOURCE_VTEP)
         {
+            disableLearningForAllVxlanNetdevices();
             m_EvpnNvoCache[EvpnNvoName] = value;
         }
     }
@@ -946,8 +956,9 @@ int VxlanMgr::createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_
 {
     std::string res, cmds;
     std::string link_add_cmd, link_set_master_cmd, link_up_cmd;
-    std::string bridge_add_cmd, bridge_untagged_add_cmd, bridge_del_vid_cmd;
+    std::string bridge_add_cmd, bridge_untagged_add_cmd, bridge_del_vid_cmd, bridge_learn_off_cmd;
     std::string vxlan_dev_name;
+    bool evpn_nvo = false;
 
     vxlan_dev_name = std::string("") + std::string(vxlanTunnelName) + "-" +
                      std::string(vlan_id);
@@ -981,11 +992,20 @@ int VxlanMgr::createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_
         SWSS_LOG_INFO("Creating VxlanNetDevice %s", vxlan_dev_name.c_str());
     }
 
+    std::map<std::string, std::string>::iterator it = m_EvpnNvoCache.begin();
+    if ((it != m_EvpnNvoCache.end()) && (it->second == vxlanTunnelName))
+    {
+        SWSS_LOG_INFO("EVPN NVO exists. Disabling learning on VxlanNetDevice %s",
+                        vxlan_dev_name.c_str());
+        evpn_nvo = true;
+    }
+
     // ip link add <vxlan_dev_name> type vxlan id <vni> local <src_ip> remote <dst_ip> 
     // dstport 4789
     // ip link set <vxlan_dev_name> master DOT1Q_BRIDGE_NAME
     // bridge vlan add vid <vlan_id> dev <vxlan_dev_name>
     // bridge vlan add vid <vlan_id> untagged pvid dev <vxlan_dev_name>
+    // bridge link set dev <vxlan_dev_name> learning off
     // ip link set <vxlan_dev_name> up
 
     link_add_cmd = std::string("") + IP_CMD + " link add " + vxlan_dev_name + 
@@ -1007,6 +1027,9 @@ int VxlanMgr::createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_
 
     bridge_del_vid_cmd = std::string("") + BRIDGE_CMD + " vlan del vid 1 dev " + 
                          vxlan_dev_name;
+
+    bridge_learn_off_cmd = std::string("") + BRIDGE_CMD + " link set dev " +
+                           vxlan_dev_name + " learning off ";
 
 
     cmds = std::string("") + BASH_CMD + " -c \"" + 
@@ -1020,6 +1043,11 @@ int VxlanMgr::createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_
         cmds += bridge_del_vid_cmd + " && ";
     }
 
+    if (evpn_nvo)
+    {
+        cmds += bridge_learn_off_cmd + " && ";
+    }
+
     cmds += link_up_cmd + "\"";
 
     return swss::exec(cmds,res);
@@ -1210,6 +1238,22 @@ void VxlanMgr::clearAllVxlanDevices()
     }
 }
 
+void VxlanMgr::disableLearningForAllVxlanNetdevices()
+{
+    for (auto it = m_vxlanTunnelMapCache.begin(); it != m_vxlanTunnelMapCache.end(); it++)
+    {
+        std::string netdev_name = it->second.vxlan_dev_name;
+        VxlanInfo info;
+        std::string res;
+        if (!netdev_name.empty())
+        {
+            SWSS_LOG_INFO("Disable learning for NetDevice %s\n", netdev_name.c_str());
+            info.m_vxlan = netdev_name;
+            cmdVxlanLearningOff(info, res);
+        }
+    }
+}
+
 void VxlanMgr::waitTillReadyToReconcile()
 {
     for (;;)

cfgmgr/vxlanmgr.h

@@ -90,6 +90,7 @@ class VxlanMgr : public Orch
     bool deleteVxlan(const VxlanInfo & info);
 
     void clearAllVxlanDevices();
+    void disableLearningForAllVxlanNetdevices();
 
     ProducerStateTable m_appVxlanTunnelTable,m_appVxlanTunnelMapTable,m_appEvpnNvoTable;
     Table m_cfgVxlanTunnelTable,m_cfgVnetTable,m_stateVrfTable,m_stateVxlanTable, m_appSwitchTable;