feat(ncs-node01,ncs-node02,ncs-wedge): Setup namespace and interface config and switch configs for l1switch

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>
This commit is contained in:
Tuan-Dat Tran 2024-12-23 16:45:18 +01:00
parent 6e8ff47cb2
commit df9c620d97
12 changed files with 456 additions and 1 deletions

View File

@ -97,7 +97,8 @@ Three folders:
- Testbed setup with alveo network cards to run on host and a switch
- Pluggable network traffic generation at the experimentation stage
- Xilinx FPGA Cards will be used
- Non static testbed - Make general first especially ARP/IP/Interface and go from there
- Non static testbed - Make general first especially ARP/IP/Interface and go
from there
### Whats needed
@ -105,6 +106,10 @@ TODO
## Plan Project
### Rough Sketch
I start by creating a rough sketch in [./sketch.md](./sketch.md).
## Implement MVP
## Refine

BIN
grafik.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 90 KiB

View File

@ -0,0 +1,55 @@
#!/bin/bash
# Run this script when:
# When the switch has been restarted or you encounter the following error:
# ```sh
# $ ./run_switchd.sh -p l1switch
# Using SDE /home/user/bf-sde-9.7.0
# Using SDE_INSTALL /home/user/bf-sde-9.7.0/install
# Setting up DMA Memory Pool
# [sudo] password for user:
# Using TARGET_CONFIG_FILE /home/user/bf-sde-9.7.0/install/share/p4/targets/tofino/l1switch.conf
# Using PATH /home/user/bf-sde-9.7.0/install/bin:/home/user/.local/bin:/home/user/bf-sde-9.7.0/install/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
# Using LD_LIBRARY_PATH /usr/local/lib:/home/user/bf-sde-9.7.0/install/lib:
# bf_sysfs_fname /sys/class/bf/bf0/device/dev_add
# Install dir: /home/user/bf-sde-9.7.0/install (0x558ddab20980)
# bf_switchd: system services initialized
# bf_switchd: loading conf_file /home/user/bf-sde-9.7.0/install/share/p4/targets/tofino/l1switch.conf...
# bf_switchd: processing device configuration...
# Configuration for dev_id 0
# Family : tofino
# pci_sysfs_str : /sys/devices/pci0000:00/0000:00:03.0/0000:05:00.0
# pci_domain : 0
# pci_bus : 5
# pci_fn : 0
# pci_dev : 0
# pci_int_mode : 1
# sbus_master_fw: /home/user/bf-sde-9.7.0/install/
# pcie_fw : /home/user/bf-sde-9.7.0/install/
# serdes_fw : /home/user/bf-sde-9.7.0/install/
# sds_fw_path : /home/user/bf-sde-9.7.0/install/share/tofino_sds_fw/avago/firmware
# microp_fw_path:
# bf_switchd: processing P4 configuration...
# P4 profile for dev_id 0
# num P4 programs 1
# p4_name: l1switch
# p4_pipeline_name: pipe
# libpd:
# libpdthrift:
# context: /home/user/bf-sde-9.7.0/install/l1switch.tofino/pipe/context.json
# config: /home/user/bf-sde-9.7.0/install/l1switch.tofino/pipe/tofino.bin
# Pipes in scope [0 1 2 3 ]
# diag:
# accton diag:
# Agent[0]: /home/user/bf-sde-9.7.0/install/lib/libpltfm_mgr.so
# non_default_port_ppgs: 0
# SAI default initialize: 1
# bf_switchd: library /home/user/bf-sde-9.7.0/install/lib/libpltfm_mgr.so loaded
# bf_switchd: agent[0] initialized
# 2024-12-09 15:37:28.067316 BF_PLTFM ERROR - Error getting ipv6 address for interface enx020000000002
#
# CHSS MGMT ERROR: Failed to configure cdc_eth ipv6
# ```
sudo "$SDE_INSTALL/bin/bf_kpkt_mod_load" "$SDE_INSTALL"
sudo ip link set "$(basename /sys/module/bf_kpkt/drivers/pci:bf/*/net/*)" up

View File

@ -0,0 +1,15 @@
#!/bin/bash
# For this code we'll need to do the following steps
# 1. `cd` into the directory where the P4 code is
# 2. Copy this script into the same directory
# 3. Run this script
# Define the name of the P4 main program as an argument to this script, like this:
# $ ./00-wedge_compile_code.sh l1switch.p4
PROGRAM=$1
COMPILED=${PROGRAM%.p4}.tofino
bf-p4c "$PROGRAM"
cp -r "$COMPILED/" "$SDE_INSTALL"
cp -r "$COMPILED"/*.conf "$SDE_INSTALL"/share/p4/targets/tofino/

View File

@ -0,0 +1,6 @@
#!/bin/bash
PROGRAM=$1
cd "$SDE" || exit
./run_switchd.sh "$PROGRAM"

View File

@ -0,0 +1,11 @@
#!/bin/bash
# from bf_shell
ucli
pm
port-add 31/- 10G NONE
an-set 31/- 2
port-enb 31/-
port-add 32/- 10G NONE
an-set 32/- 2
port-enb 32/-

View File

@ -0,0 +1,28 @@
# from bf_shell
bfrt_python
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=266, dst_addr="10.0.1.1"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=267, dst_addr="10.0.1.2"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=265, dst_addr="10.0.1.3"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=264, dst_addr="10.0.1.4"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=256, dst_addr="10.0.2.1"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=258, dst_addr="10.0.2.2"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=257, dst_addr="10.0.2.3"
)
bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send(
egress_port=258, dst_addr="10.0.2.4"
)
bfrt.complete_operations()

108
scripts/10-host_notes.sh Normal file
View File

@ -0,0 +1,108 @@
#!/bin/bash
# ncs-node1: enp6s0f0np0 / enp6s0f1np1
# - Control:
# - enp4s0f2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
# - Data:
# - enp6s0f0np0 | 10.0.1.1 | 64:9d:99:b1:0a:88 | tb_node1_if0 | 32/2 | 266
# - enp6s0f1np1 | 10.0.1.2 | 64:9d:99:b1:0a:89 | tb_node1_if1 | 32/3 | 267
# - enp6s0f2np2 | 10.0.1.3 | 64:9d:99:b1:0a:8a | tb_node1_if2 | 32/1 | 265
# - enp6s0f3np3 | 10.0.1.4 | 64:9d:99:b1:0a:8b | tb_node1_if3 | 32/0 | 264
#
# ncs-node2:
# - Control:
# - enp4s0f2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
# - Data:
# - enp6s0f0np0 | 10.0.2.1 | 64:9d:99:b1:0a:a4 | tb_node2_if0 | 31/0 | 256
# - enp6s0f1np1 | 10.0.2.2 | 64:9d:99:b1:0a:a5 | tb_node2_if1 | 31/3 | 259
# - enp6s0f2np2 | 10.0.2.3 | 64:9d:99:b1:0a:a6 | tb_node2_if2 | 31/1 | 257
# - enp6s0f3np3 | 10.0.2.4 | 64:9d:99:b1:0a:a7 | tb_node2_if3 | 31/2 | 258
#
# ncs-wedge:
# PORT |MAC |D_P|P/PT|SPEED |FEC |AN|KR|RDY|ADM|OPR|LPBK |FRAMES RX |FRAMES TX |E
# 31/0 |48/0|256|2/ 0|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 62| 0|
# 31/1 |48/1|257|2/ 1|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 62| 0|
# 31/2 |48/2|258|2/ 2|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 62| 0|
# 31/3 |48/3|259|2/ 3|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 62| 0|
# 32/0 |50/0|264|2/ 8|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 62| 0|
# 32/1 |50/1|265|2/ 9|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 62| 0|
# 32/2 |50/2|266|2/10|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 65| 0|
# 32/3 |50/3|267|2/11|10G |NONE|Ds|Au|YES|ENB|UP | NONE | 65| 0|
# node1
# Create Namespace: Testbed on Node1 for Interface n
sudo ip netns add tb_node1_if0
sudo ip netns add tb_node1_if1
sudo ip netns add tb_node1_if2
sudo ip netns add tb_node1_if3
# Add interfaces to namespaces
sudo ip link set enp6s0f0np0 netns tb_node1_if0
sudo ip link set enp6s0f1np1 netns tb_node1_if1
sudo ip link set enp6s0f2np2 netns tb_node1_if2
sudo ip link set enp6s0f3np3 netns tb_node1_if3
# Show Info
sudo ip netns exec tb_node1_if0 ip a s enp6s0f0np0
sudo ip netns exec tb_node1_if1 ip a s enp6s0f1np1
sudo ip netns exec tb_node1_if2 ip a s enp6s0f2np2
sudo ip netns exec tb_node1_if3 ip a s enp6s0f3np3
# Set IP Addresses
sudo ip netns exec tb_node1_if0 ip addr add dev enp6s0f0np0 10.0.1.1/16
sudo ip netns exec tb_node1_if1 ip addr add dev enp6s0f1np1 10.0.1.2/16
sudo ip netns exec tb_node1_if2 ip addr add dev enp6s0f2np2 10.0.1.3/16
sudo ip netns exec tb_node1_if3 ip addr add dev enp6s0f3np3 10.0.1.4/16
# Turn on Links
sudo ip netns exec tb_node1_if0 ip link set up enp6s0f0np0
sudo ip netns exec tb_node1_if1 ip link set up enp6s0f1np1
sudo ip netns exec tb_node1_if2 ip link set up enp6s0f2np2
sudo ip netns exec tb_node1_if3 ip link set up enp6s0f3np3
sudo ./11-host_arps.sh tb_node1_if0 enp6s0f0np0
sudo ./11-host_arps.sh tb_node1_if1 enp6s0f1np1
sudo ./11-host_arps.sh tb_node1_if2 enp6s0f2np2
sudo ./11-host_arps.sh tb_node1_if3 enp6s0f3np3
# node2
# Create Namespace: Testbed on Node1 for Interface n
sudo ip netns add tb_node2_if0
sudo ip netns add tb_node2_if1
sudo ip netns add tb_node2_if2
sudo ip netns add tb_node2_if3
# Add interfaces to namespaces
sudo ip link set enp6s0f0np0 netns tb_node2_if0
sudo ip link set enp6s0f1np1 netns tb_node2_if1
sudo ip link set enp6s0f2np2 netns tb_node2_if2
sudo ip link set enp6s0f3np3 netns tb_node2_if3
# Show Info
sudo ip netns exec tb_node2_if0 ip a s enp6s0f0np0
sudo ip netns exec tb_node2_if1 ip a s enp6s0f1np1
sudo ip netns exec tb_node2_if2 ip a s enp6s0f2np2
sudo ip netns exec tb_node2_if3 ip a s enp6s0f3np3
# Set IP Addresses
sudo ip netns exec tb_node2_if0 ip addr add dev enp6s0f0np0 10.0.2.1/16
sudo ip netns exec tb_node2_if1 ip addr add dev enp6s0f1np1 10.0.2.2/16
sudo ip netns exec tb_node2_if2 ip addr add dev enp6s0f2np2 10.0.2.3/16
sudo ip netns exec tb_node2_if3 ip addr add dev enp6s0f3np3 10.0.2.4/16
# Turn on Links
sudo ip netns exec tb_node2_if0 ip link set up enp6s0f0np0
sudo ip netns exec tb_node2_if1 ip link set up enp6s0f1np1
sudo ip netns exec tb_node2_if2 ip link set up enp6s0f2np2
sudo ip netns exec tb_node2_if3 ip link set up enp6s0f3np3
# execute arp
sudo ./11-host_arps.sh tb_node2_if0 enp6s0f0np0
sudo ./11-host_arps.sh tb_node2_if1 enp6s0f1np1
sudo ./11-host_arps.sh tb_node2_if2 enp6s0f2np2
sudo ./11-host_arps.sh tb_node2_if3 enp6s0f3np3
# Show arp
sudo ip netns exec tb_node2_if0 ip a s enp6s0f0np0
sudo ip netns exec tb_node2_if1 ip a s enp6s0f1np1
sudo ip netns exec tb_node2_if2 ip a s enp6s0f2np2
sudo ip netns exec tb_node2_if3 ip a s enp6s0f3np3

14
scripts/11-host_arps.sh Normal file
View File

@ -0,0 +1,14 @@
#!/bin/bash
NS=$1
IF=$2
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.1.1 64:9d:99:b1:0a:88
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.1.2 64:9d:99:b1:0a:89
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.1.3 64:9d:99:b1:0a:8a
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.1.4 64:9d:99:b1:0a:8b
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.2.1 64:9d:99:b1:0a:a4
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.2.2 64:9d:99:b1:0a:a5
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.2.3 64:9d:99:b1:0a:a6
sudo ip netns exec "$NS" arp -i "$IF" -s 10.0.2.4 64:9d:99:b1:0a:a7

View File

@ -0,0 +1,67 @@
import sys
import time
import threading
from scapy.all import Ether, IP, UDP, send, sendp, Raw, sniff
global initial_timestamp
initial_timestamp = int(time.time() * 1000)
# Simulated ACK sending function
def send_ack(destination_IP, my_port, sender_port):
ack_data = ""
###################
# With localhost:
ack_packet = IP(dst=destination_IP) / UDP(dport=sender_port, sport=my_port) / Raw(load=str(ack_data))
send(ack_packet, verbose=False)
###################
# With netns:
# ack_packet = Ether(dst="00:00:00:00:00:04", src="00:00:00:00:00:01") / IP(dst="127.0.0.1") / UDP(dport=5551) / Raw(load=str(ack_data))
# sendp(ack_packet, verbose=False)
###################
print(f"Sent ACK: {ack_data}")
# Process incoming packets
def process_packet(pkt):
################### Save timestamps ###################
if Raw in pkt:
with open(save_file, "a") as f: # open as "a" for append
global initial_timestamp
receival_time = str(int(time.time() * 1000) - initial_timestamp) # time of arrival at receiver
send_time = pkt[Raw].load # time it was sent from the sender
################### Conversion & error handling ###################
try:
send_time = int(send_time.decode("utf-8").split("X")[0]) # decode the payload and stop at "X"
except ValueError: # For the last packet without a timestamp but "END" as payload
print(f"END for {my_port}")
send_ack(destination_IP, my_port, sender_port)
return
except Exception as e:
print(f"Decoding error at {my_port}")
send_ack(destination_IP, my_port, sender_port)
return
################### Write data ###################
aoi = int(receival_time) - send_time
f.write(f"{send_time},{receival_time},{aoi}\n")
send_ack(destination_IP, my_port, sender_port)
# Main loop
def main(destination_IP, my_port, sender_port):
###################
# With localhost:
sniff(iface="lo", filter=f"udp and dst port {my_port}", prn=process_packet, store=False)
###################
# With netns:
# sniff(iface="ns-g", filter=f"udp and dst port {my_port}", prn=process_packet, store=False)
###################
if __name__ == "__main__":
if len(sys.argv) != 4:
print("Usage: python receive_and_ack.py <destination_IP> <my_port> <sender_port>")
sys.exit(1)
destination_IP = sys.argv[1] # Clear
my_port = int(sys.argv[2]) # Port to listen on
sender_port = int(sys.argv[3]) # Port to send acks to
save_file = f"timestamps_{my_port}"
main(destination_IP, my_port, sender_port)

View File

@ -0,0 +1,134 @@
import sys
import time
import random
import threading
from scapy.all import Ether, IP, UDP, sendp, Raw, sniff, send
global initial_timestamp
initial_timestamp = int(time.time() * 1000)
# Parameters
Q_max = 10 # Maximum queue size
age_threshold = 1.0 # Delta_T in seconds
# Sender's state
last_update_time = None # Last time an ACK was received
N = 1 # Active senders (received from ACK)
queue_utilization = 0 # Received as Q_i/Q_max
delta_c = 0 # Time since the last ACK (calculated using T_ACK)
# Function to calculate P_s
def calculate_probability(N, queue_utilization, delta_c):
if queue_utilization < 1:
return 1.0 # Full probability if the queue is underutilized
elif queue_utilization == 1:
# Calculate f(Delta_i)
f_delta = max((delta_c - age_threshold) / age_threshold, 0)
return (Q_max / N) + f_delta # Combine probabilities
# Function to send a packet
def send_packet(target_ip, receiver_port, my_port, dscp_value, packet_size, FIN_flag):
ether_layer = Ether(dst="00:00:00:00:00:04", src="00:00:00:00:00:01")
ip_layer = IP(dst=target_ip, tos=dscp_value << 2)
udp_layer = UDP(dport=receiver_port, sport=my_port)
# # Socket to listen for the ACKs
# ack_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# ack_socket.settimeout(2) # 2 seconds timer to receive the ack. Otherwise considered lost
# ack_socket.bind(("", my_port))
# timestamp = str(int(time.time())) # Convert to a string for embedding
global initial_timestamp
timestamp = str(int(time.time() * 1000) - initial_timestamp)
if (not FIN_flag):
payload = timestamp + "X" * max((packet_size - len(timestamp)), 0) # max to prevent possible errors when choosing too small packet size
else:
payload = "END" + "X" * max((packet_size - len(timestamp)), 0)
# packet = ether_layer / ip_layer / udp_layer / Raw(load=payload)
packet = ip_layer / udp_layer / Raw(load=payload)
# sendp(packet, verbose=False)
send(packet, verbose=False)
print(".", end='', flush=True)
# Listen for ACKs
def process_ack(pkt):
global last_update_time, N, queue_utilization, delta_c
if Raw in pkt:
ack_data = pkt[Raw].load.decode('utf-8')
N, queue_utilization, T_ACK = eval(ack_data)
last_update_time = T_ACK
print(f"Received ACK: N={N}, Utilization={queue_utilization}, T_ACK={T_ACK}")
# Main loop
def main(target_ip, receiver_port, my_port, dscp_value, nb_packets, packet_size):
global delta_c
packets_sent = 0
while packets_sent < nb_packets:
# Update delta_c (time since the last ACK)
if last_update_time:
delta_c = time.time() - last_update_time
else:
delta_c = float('inf') # No ACK received yet
# Calculate sending probability
P_s = calculate_probability(N, queue_utilization, delta_c)
# Decide whether to send a packet
if random.random() < P_s:
FIN_flag = False
if packets_sent == nb_packets - 1:
FIN_flag = True
send_packet(target_ip, receiver_port, my_port, dscp_value, packet_size, FIN_flag)
packets_sent += 1
print(f"Packet sent with probability P_s={P_s:.2f}")
else:
print(f"Packet skipped with probability P_s={P_s:.2f}")
# Wait for a short interval before the next attempt
time.sleep(0.1)
print(f"Finished sending {nb_packets} packets to {target_ip}:{receiver_port}")
if __name__ == "__main__":
if len(sys.argv) != 6:
print("Usage: python send_dscp_packets.py <target_ip> <my_port> <receiver_port> <nb_packets> <packet_size(in Bytes)>")
sys.exit(1)
target_ip = sys.argv[1] # IP of the receiver
my_port = int(sys.argv[2]) # Sender port
receiver_port = int(sys.argv[3]) # Receiver port
nb_packets = int(sys.argv[4]) # Total number of packets to send
packet_size = int(sys.argv[5]) - 42 # Packet size in Bytes - 42 (previous headers)
dscp_value = 0 # Hardcoded DSCP value for now
# main(target_ip, receiver_port, my_port, dscp_value, nb_packets)
threading.Thread(target=main, args=(target_ip, receiver_port, my_port, dscp_value, nb_packets, packet_size), daemon=True).start()
###################
# With localhost:
sniff(iface="lo", filter=f"udp and dst port {my_port}", prn=process_ack, store=False)#, timeout=0.1)
###################
# With netns:
# sniff(iface="ns-r", filter=f"udp and dst port {my_port}", prn=process_ack, store=False)
###################
''' Archive:
# Add padding:
payload = timestamp + "X" * (packet_size - len(timestamp))
'''

12
sketch.md Normal file
View File

@ -0,0 +1,12 @@
# Setup of P4 Testbed
| RP-ECN | Patricks |
| -------------------------------------------------------------------- | --------------------------------------- |
| Setup ARP on each host | |
| (When first time) Install required tools | Set time on each node |
| Run switch program on wedge | Setup Namespaces on node |
| Prepare Ports on wedge | Setup Interfaces/IPs on node |
| Configure touting tables on wedge | Setup Routing on node |
| Do Portshaping | ARP Entries on node |
| Run experiment-script | |
| Reset of setup can be done by executing `reset.sh` on each test node | Delete all namespaces on each test node |