From 5857beeaf66dda4f7efe40ecbcf9362ccb1e60c6 Mon Sep 17 00:00:00 2001 From: Tuan-Dat Tran Date: Tue, 31 Dec 2024 15:17:29 +0100 Subject: [PATCH] feat(automation/deployment): Added script to autodeploy and setup wedge and nodes Signed-off-by: Tuan-Dat Tran --- README.md | 119 +--------------------------- scripts/00-wedge_on_reboot.sh | 6 +- scripts/01-wedge_compile_code.sh | 7 +- scripts/02-run_switch_program.sh | 4 +- scripts/03-wedge_port_configure.sh | 11 --- scripts/04-wedge_route_configure.py | 2 - scripts/10-host_setup_node1.sh | 0 scripts/10-host_setup_node2.sh | 0 scripts/19-cleanup.sh | 0 scripts/30-add_namespaces.sh | 0 scripts/31-configure_interfaces.sh | 12 --- scripts/32-setup_arp.sh | 0 scripts/98-test.sh | 7 ++ scripts/99-deployment.sh | 68 ++++++++++++++++ scripts/README.md | 3 + scripts/l1switch.service | 13 +++ 16 files changed, 104 insertions(+), 148 deletions(-) mode change 100644 => 100755 scripts/00-wedge_on_reboot.sh mode change 100644 => 100755 scripts/01-wedge_compile_code.sh mode change 100644 => 100755 scripts/02-run_switch_program.sh delete mode 100644 scripts/03-wedge_port_configure.sh mode change 100644 => 100755 scripts/10-host_setup_node1.sh mode change 100644 => 100755 scripts/10-host_setup_node2.sh mode change 100644 => 100755 scripts/19-cleanup.sh mode change 100644 => 100755 scripts/30-add_namespaces.sh mode change 100644 => 100755 scripts/31-configure_interfaces.sh mode change 100644 => 100755 scripts/32-setup_arp.sh create mode 100755 scripts/98-test.sh create mode 100755 scripts/99-deployment.sh create mode 100644 scripts/l1switch.service diff --git a/README.md b/README.md index 0f69b6b..bd6ac4d 100644 --- a/README.md +++ b/README.md @@ -1,117 +1,4 @@ -# Notes for "P4 Experiment Setup" +# P4 Experiment Setup -1. [ ] Review -2. [ ] Requirements -3. [ ] Plan Project -4. [ ] Implement MVP (Minimum Viable Product) -5. [ ] Refine -6. [ ] Documentation - -## Review - -### Patricks Code - -#### Patrick Review Notes - -Everything in `dev` branch. - -[picoquic](https://github.com/private-octopus/picoquic) is a minimalist QUIC -implementation Patrick used to conduct his experiments with QUIC. - -The wedge is the P4 Switch Patrick used for testing on hardware. -It is pre-configured to run P4 Code. - -Test nodes are: - -- ncs-node3-tailscale -- ncs-node4-tailscale -- ncs-node5-tailscale -- ncs-node6-tailscale -- ncs-node7-tailscale - -Three folders: - -- `bqm`: - - `bqm`: Implementation of bqm along with a compilation script - - `config`: scripts to configure bfrt - automates interactions with bfshell - - `taildrop`: Drop bqm control packets and if queue is full (INCOMPLETE EXPLANATION) - - `test`: Testing scripted setup of a network namespace on one host - - `clean_env.sh`: Delete all network namespaces - - `setup_env.sh`: Setup NS `exp_server` and `exp_client`, add interfaces/IP/ARP - entries - - `start_tshark_1.sh`: Runs `tshark` in the network namespace - `exp_server` on `eno1np0` - - `test.py`: Use scapy to send 20 UDP packets - - `test_switch`: P4 Script and BFRT configuration for testing - - `copy_all.sh`: Script to copy bqm and config to the Wedge -- `experiment-scripts`: - - `experiment_set_1`: Experiment. Explanation in `README.md` - - `experiment_set_2`: Experiment. Explanation in `README.md` - - `experiment_set_3`: Experiment. Explanation in `README.md` - - `experiment_set_4`: Experiment. Explanation in `README.md` - - `experiment_set_reno1`: Experiment. Explanation in `README.md` - - `experiment_set_reno2`: Experiment. Explanation in `README.md` - - `experiment_set_reno3`: Experiment. Explanation in `README.md` - - `experiment_set_reno4`: Experiment. Explanation in `README.md` - - `results`: Results of prior experiments. - - `cleanup_network.sh`: Delete all namespaces on each test node - - `configure_picoquic.sh`: Compile picoquic on each test node - - `create_folders.sh`: Create experiment folders - - `credentials`: Git credentials - - `diagrams.ipynb`: - - `old.ipynb`: - - `run_sudo.sh`: Helper script to run sudo commands via ssh - - `set_arp_entries.sh`: Set ARP entries of test setup - - `setup_log_locations.sh`: Copy and execute `create_folders.sh`-script on - each test node - - `setup_network.sh`: Setup test network. Includes: - - Namespaces - - Interfaces/IPs - - Routing - - ARP Entries (with `set_arp_entries.sh`) - - `setup_repositories.sh`: Setup git to get picoquic on each test node - - `sync_time.sh`: Set time on each node - - `update_repositories.sh`: Update picoquic on each test node -- `picoquic-sync`: Local copy of the picoquic library - -### RECN Code - -#### 04\_-_Projekte/14-run_rpecn_tests/codes/Big_Topo - -- `README.md`: - - Explains the setup and prerequisite steps to run an experiment - - Prerequisite Steps: - - Setup ARP on each host - - (When first time) Install required tools - - Run switch program on wedge - - Prepare Ports on wedge - - Configure touting tables on wedge - - Do Portshaping - - Run experiment-script - - Reset of setup can be done by executing `reset.sh` on each test node - -## Requirements - -### Notes - -- Testbed setup with alveo network cards to run on host and a switch -- Pluggable network traffic generation at the experimentation stage -- Xilinx FPGA Cards will be used -- Non static testbed - Make general first especially ARP/IP/Interface and go - from there - -### Whats needed - -TODO - -## Plan Project - -### Rough Sketch - -I start by creating a rough sketch in [./sketch.md](./sketch.md). - -## Implement MVP - -## Refine - -## Documentation +Look at [./scripts/README.md](./scripts/README.md) for more details about the +topology and how to get started. diff --git a/scripts/00-wedge_on_reboot.sh b/scripts/00-wedge_on_reboot.sh old mode 100644 new mode 100755 index 131c655..b32539f --- a/scripts/00-wedge_on_reboot.sh +++ b/scripts/00-wedge_on_reboot.sh @@ -1,5 +1,7 @@ #!/bin/bash +SDE_INSTALL=/home/user/bf-sde-9.7.0/install + # Run this script when: # When the switch has been restarted or you encounter the following error: # ```sh @@ -51,5 +53,5 @@ # CHSS MGMT ERROR: Failed to configure cdc_eth ipv6 # ``` -sudo "$SDE_INSTALL/bin/bf_kpkt_mod_load" "$SDE_INSTALL" -sudo ip link set "$(basename /sys/module/bf_kpkt/drivers/pci:bf/*/net/*)" up +"$SDE_INSTALL/bin/bf_kpkt_mod_load" "$SDE_INSTALL" +ip link set "$(basename /sys/module/bf_kpkt/drivers/pci:bf/*/net/*)" up diff --git a/scripts/01-wedge_compile_code.sh b/scripts/01-wedge_compile_code.sh old mode 100644 new mode 100755 index a397b7f..030f6b3 --- a/scripts/01-wedge_compile_code.sh +++ b/scripts/01-wedge_compile_code.sh @@ -1,5 +1,7 @@ #!/bin/bash +SDE_INSTALL=/home/user/bf-sde-9.7.0/install + # For this code we'll need to do the following steps # 1. `cd` into the directory where the P4 code is # 2. Copy this script into the same directory @@ -7,9 +9,10 @@ # Define the name of the P4 main program as an argument to this script, like this: # $ ./00-wedge_compile_code.sh l1switch.p4 -PROGRAM=$1 +PROGRAM=l1switch.p4 COMPILED=${PROGRAM%.p4}.tofino -bf-p4c "$PROGRAM" +cd /tmp/l1switch || exit +/home/user/bf-sde-9.7.0/install/bin/bf-p4c "$PROGRAM" 1>/dev/null cp -r "$COMPILED/" "$SDE_INSTALL" cp -r "$COMPILED"/*.conf "$SDE_INSTALL"/share/p4/targets/tofino/ diff --git a/scripts/02-run_switch_program.sh b/scripts/02-run_switch_program.sh old mode 100644 new mode 100755 index 2ef14ab..7fc008d --- a/scripts/02-run_switch_program.sh +++ b/scripts/02-run_switch_program.sh @@ -1,6 +1,4 @@ #!/bin/bash -PROGRAM=$1 - cd "$SDE" || exit -./run_switchd.sh "$PROGRAM" +SDE=/home/user/bf-sde-9.7.0/ SDE_INSTALL=/home/user/bf-sde-9.7.0/install ./run_switchd.sh -p l1switch diff --git a/scripts/03-wedge_port_configure.sh b/scripts/03-wedge_port_configure.sh deleted file mode 100644 index bcf4472..0000000 --- a/scripts/03-wedge_port_configure.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# from bf_shell -ucli -pm -port-add 31/- 10G NONE -an-set 31/- 2 -port-enb 31/- -port-add 32/- 10G NONE -an-set 32/- 2 -port-enb 32/- diff --git a/scripts/04-wedge_route_configure.py b/scripts/04-wedge_route_configure.py index 0db8c54..15b94e0 100644 --- a/scripts/04-wedge_route_configure.py +++ b/scripts/04-wedge_route_configure.py @@ -1,5 +1,3 @@ -# from bf_shell -# bfrt_python bfrt.l1switch.pipe.SwitchIngress.t_l1_forwarding.add_with_send( egress_port=266, dst_addr="10.0.1.1" ) diff --git a/scripts/10-host_setup_node1.sh b/scripts/10-host_setup_node1.sh old mode 100644 new mode 100755 diff --git a/scripts/10-host_setup_node2.sh b/scripts/10-host_setup_node2.sh old mode 100644 new mode 100755 diff --git a/scripts/19-cleanup.sh b/scripts/19-cleanup.sh old mode 100644 new mode 100755 diff --git a/scripts/30-add_namespaces.sh b/scripts/30-add_namespaces.sh old mode 100644 new mode 100755 diff --git a/scripts/31-configure_interfaces.sh b/scripts/31-configure_interfaces.sh old mode 100644 new mode 100755 index 0fa05c4..e6e7807 --- a/scripts/31-configure_interfaces.sh +++ b/scripts/31-configure_interfaces.sh @@ -6,26 +6,14 @@ IP=$3 # Add interfaces to namespaces ip link set "$IF" netns "$NS" -ip link set "$IF" netns "$NS" -ip link set "$IF" netns "$NS" -ip link set "$IF" netns "$NS" # Set IP Addresses ip netns exec "$NS" ip addr add dev "$IF" "$IP" -ip netns exec "$NS" ip addr add dev "$IF" "$IP" -ip netns exec "$NS" ip addr add dev "$IF" "$IP" -ip netns exec "$NS" ip addr add dev "$IF" "$IP" # Turn on Links ip netns exec "$NS" ip link set up "$IF" -ip netns exec "$NS" ip link set up "$IF" -ip netns exec "$NS" ip link set up "$IF" -ip netns exec "$NS" ip link set up "$IF" # For Debugging # # # Show interface info # ip netns exec "$NS" ip a s "$IF" -# ip netns exec "$NS" ip a s "$IF" -# ip netns exec "$NS" ip a s "$IF" -# ip netns exec "$NS" ip a s "$IF" diff --git a/scripts/32-setup_arp.sh b/scripts/32-setup_arp.sh old mode 100644 new mode 100755 diff --git a/scripts/98-test.sh b/scripts/98-test.sh new file mode 100755 index 0000000..b6bc54e --- /dev/null +++ b/scripts/98-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SDE=/home/user/bf-sde-9.7.0/ +SDE_INSTALL=/home/user/bf-sde-9.7.0/install + +echo "$SDE" +echo "$SDE_INSTALL" diff --git a/scripts/99-deployment.sh b/scripts/99-deployment.sh new file mode 100755 index 0000000..5520ea6 --- /dev/null +++ b/scripts/99-deployment.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +WEDGE_PASS="Jony154977" +USER_PASS="asdf1234" + +run_script_via_ssh() { + local script_path="$1" # Path to the script to be copied + local ssh_host="$2" # SSH host + local pass=$3 + local remote_path="/tmp/$(basename "$script_path")" # Destination path on the remote machine + + # Copy the script to the remote host + scp "$script_path" "$ssh_host:$remote_path" + if [[ $? -ne 0 ]]; then + echo "Error: Failed to copy the script to the remote host." + return 1 + fi + + # Execute the script on the remote host with sudo + ssh "$ssh_host" "echo $pass | sudo -S $remote_path" + if [[ $? -ne 0 ]]; then + echo "Error: Failed to execute the script on the remote host." + return 1 + fi + + echo "Script executed successfully on $ssh_host." +} + +wedge_setup() { + scp -r ../l1switch/ ncs-wedge:/tmp/ + run_script_via_ssh ./00-wedge_on_reboot.sh ncs-wedge $WEDGE_PASS + run_script_via_ssh ./01-wedge_compile_code.sh ncs-wedge $WEDGE_PASS + scp ./l1switch.service ncs-wedge:/tmp/l1switch.service + scp ./02-run_switch_program.sh ncs-wedge:/home/user/bf-sde-9.7.0/l1switch.sh + ssh ncs-wedge "echo $WEDGE_PASS | sudo -S mv /tmp/l1switch.service /etc/systemd/system/l1switch.service" + ssh ncs-wedge "echo $WEDGE_PASS | sudo -S systemctl daemon-reload" + ssh ncs-wedge "echo $WEDGE_PASS | sudo -S systemctl start l1switch.service" + + scp ./03-wedge_port_configure.command ncs-wedge:/tmp/ + ssh ncs-wedge "/home/user/bf-sde-9.7.0/install/bin/bfshell -f /tmp/03-wedge_port_configure.command" + + scp ./04-wedge_route_configure.py ncs-wedge:/tmp/ + ssh ncs-wedge "/home/user/bf-sde-9.7.0/install/bin/bfshell -b /tmp/04-wedge_route_configure.py" +} + +setup_node() { + local nodeid="$1" + local node="ncs-node$nodeid" + local node_script="./10-host_setup_node$nodeid.sh" + + scp "./30-add_namespaces.sh" "$node": + scp "./31-configure_interfaces.sh" "$node": + scp "./32-setup_arp.sh" "$node": + + run_script_via_ssh "$node_script" "$node" $USER_PASS +} + +# wedge_setup +# setup_node 1 +# setup_node 2 + +read -p "YOU WILL NEED TO PRESS CTRL+C AT SOME POINT SINCE THE RECEIVER WILL RUN INDEFINETLY. (Press any key to confirm.)" -n1 -s + +TIMEOUT=30 +ssh ncs-node1 "echo $USER_PASS | sudo -S timeout $TIMEOUT ip netns exec tb_node1_if0 python3 receiver.py 10.0.2.1 30123 30123" & +ssh ncs-node2 "echo $USER_PASS | sudo -S timeout $TIMEOUT ip netns exec tb_node2_if0 python3 sender.py 10.0.1.1 30123 30123 100 128" + +scp ncs-node1:timestamps_30123 . diff --git a/scripts/README.md b/scripts/README.md index 94e3cce..58cf44d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -3,6 +3,9 @@ In this repository we're have scripts to setup a network testbed with 2 nodes and a Tofino Wedge 100BF-32X 100GBE Baremetal Switch. +You can run the script [./99-deployment.sh](./99-deployment.sh) for a sample +deployment. + ## Prerequisites - Hardware diff --git a/scripts/l1switch.service b/scripts/l1switch.service new file mode 100644 index 0000000..a9118c0 --- /dev/null +++ b/scripts/l1switch.service @@ -0,0 +1,13 @@ +[Unit] +Description=Run L1Switch Program +After=network.target + +[Service] +ExecStart=/home/user/bf-sde-9.7.0/l1switch.sh +WorkingDirectory=/home/user/bf-sde-9.7.0 +User=root +Group=root +Restart=always + +[Install] +WantedBy=multi-user.target