1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Author: Justin Iurman <justin.iurman@uliege.be> 5# 6# This script evaluates the IOAM insertion for IPv6 by checking the IOAM data 7# consistency directly inside packets on the receiver side. Tests are divided 8# into three categories: OUTPUT (evaluates the IOAM processing by the sender), 9# INPUT (evaluates the IOAM processing by the receiver) and GLOBAL (evaluates 10# wider use cases that do not fall into the other two categories). Both OUTPUT 11# and INPUT tests only use a two-node topology (alpha and beta), while GLOBAL 12# tests use the entire three-node topology (alpha, beta, gamma). Each test is 13# documented inside its own handler in the code below. 14# 15# An IOAM domain is configured from Alpha to Gamma but not on the reverse path. 16# When either Beta or Gamma is the destination (depending on the test category), 17# Alpha adds an IOAM option (Pre-allocated Trace) inside a Hop-by-hop. 18# 19# 20# +-------------------+ +-------------------+ 21# | | | | 22# | Alpha netns | | Gamma netns | 23# | | | | 24# | +-------------+ | | +-------------+ | 25# | | veth0 | | | | veth0 | | 26# | | db01::2/64 | | | | db02::2/64 | | 27# | +-------------+ | | +-------------+ | 28# | . | | . | 29# +-------------------+ +-------------------+ 30# . . 31# . . 32# . . 33# +----------------------------------------------------+ 34# | . . | 35# | +-------------+ +-------------+ | 36# | | veth0 | | veth1 | | 37# | | db01::1/64 | ................ | db02::1/64 | | 38# | +-------------+ +-------------+ | 39# | | 40# | Beta netns | 41# | | 42# +----------------------------------------------------+ 43# 44# 45# 46# ============================================================= 47# | Alpha - IOAM configuration | 48# +===========================================================+ 49# | Node ID | 1 | 50# +-----------------------------------------------------------+ 51# | Node Wide ID | 11111111 | 52# +-----------------------------------------------------------+ 53# | Ingress ID | 0xffff (default value) | 54# +-----------------------------------------------------------+ 55# | Ingress Wide ID | 0xffffffff (default value) | 56# +-----------------------------------------------------------+ 57# | Egress ID | 101 | 58# +-----------------------------------------------------------+ 59# | Egress Wide ID | 101101 | 60# +-----------------------------------------------------------+ 61# | Namespace Data | 0xdeadbee0 | 62# +-----------------------------------------------------------+ 63# | Namespace Wide Data | 0xcafec0caf00dc0de | 64# +-----------------------------------------------------------+ 65# | Schema ID | 777 | 66# +-----------------------------------------------------------+ 67# | Schema Data | something that will be 4n-aligned | 68# +-----------------------------------------------------------+ 69# 70# 71# ============================================================= 72# | Beta - IOAM configuration | 73# +===========================================================+ 74# | Node ID | 2 | 75# +-----------------------------------------------------------+ 76# | Node Wide ID | 22222222 | 77# +-----------------------------------------------------------+ 78# | Ingress ID | 201 | 79# +-----------------------------------------------------------+ 80# | Ingress Wide ID | 201201 | 81# +-----------------------------------------------------------+ 82# | Egress ID | 202 | 83# +-----------------------------------------------------------+ 84# | Egress Wide ID | 202202 | 85# +-----------------------------------------------------------+ 86# | Namespace Data | 0xdeadbee1 | 87# +-----------------------------------------------------------+ 88# | Namespace Wide Data | 0xcafec0caf11dc0de | 89# +-----------------------------------------------------------+ 90# | Schema ID | 666 | 91# +-----------------------------------------------------------+ 92# | Schema Data | Hello there -Obi | 93# +-----------------------------------------------------------+ 94# 95# 96# ============================================================= 97# | Gamma - IOAM configuration | 98# +===========================================================+ 99# | Node ID | 3 | 100# +-----------------------------------------------------------+ 101# | Node Wide ID | 33333333 | 102# +-----------------------------------------------------------+ 103# | Ingress ID | 301 | 104# +-----------------------------------------------------------+ 105# | Ingress Wide ID | 301301 | 106# +-----------------------------------------------------------+ 107# | Egress ID | 0xffff (default value) | 108# +-----------------------------------------------------------+ 109# | Egress Wide ID | 0xffffffff (default value) | 110# +-----------------------------------------------------------+ 111# | Namespace Data | 0xdeadbee2 | 112# +-----------------------------------------------------------+ 113# | Namespace Wide Data | 0xcafec0caf22dc0de | 114# +-----------------------------------------------------------+ 115# | Schema ID | 0xffffff (= None) | 116# +-----------------------------------------------------------+ 117# | Schema Data | | 118# +-----------------------------------------------------------+ 119 120 121################################################################################ 122# # 123# WARNING: Be careful if you modify the block below - it MUST be kept # 124# synchronized with configurations inside ioam6_parser.c and always # 125# reflect the same. # 126# # 127################################################################################ 128 129ALPHA=( 130 1 # ID 131 11111111 # Wide ID 132 0xffff # Ingress ID 133 0xffffffff # Ingress Wide ID 134 101 # Egress ID 135 101101 # Egress Wide ID 136 0xdeadbee0 # Namespace Data 137 0xcafec0caf00dc0de # Namespace Wide Data 138 777 # Schema ID (0xffffff = None) 139 "something that will be 4n-aligned" # Schema Data 140) 141 142BETA=( 143 2 144 22222222 145 201 146 201201 147 202 148 202202 149 0xdeadbee1 150 0xcafec0caf11dc0de 151 666 152 "Hello there -Obi" 153) 154 155GAMMA=( 156 3 157 33333333 158 301 159 301301 160 0xffff 161 0xffffffff 162 0xdeadbee2 163 0xcafec0caf22dc0de 164 0xffffff 165 "" 166) 167 168TESTS_OUTPUT=" 169 out_undef_ns 170 out_no_room 171 out_bits 172 out_full_supp_trace 173" 174 175TESTS_INPUT=" 176 in_undef_ns 177 in_no_room 178 in_oflag 179 in_bits 180 in_full_supp_trace 181" 182 183TESTS_GLOBAL=" 184 fwd_full_supp_trace 185" 186 187 188################################################################################ 189# # 190# LIBRARY # 191# # 192################################################################################ 193 194check_kernel_compatibility() 195{ 196 ip netns add ioam-tmp-node 197 ip link add name veth0 netns ioam-tmp-node type veth \ 198 peer name veth1 netns ioam-tmp-node 199 200 ip -netns ioam-tmp-node link set veth0 up 201 ip -netns ioam-tmp-node link set veth1 up 202 203 ip -netns ioam-tmp-node ioam namespace add 0 &>/dev/null 204 ns_ad=$? 205 206 ip -netns ioam-tmp-node ioam namespace show | grep -q "namespace 0" 207 ns_sh=$? 208 209 if [[ $ns_ad != 0 || $ns_sh != 0 ]] 210 then 211 echo "SKIP: kernel version probably too old, missing ioam support" 212 ip link del veth0 2>/dev/null || true 213 ip netns del ioam-tmp-node || true 214 exit 1 215 fi 216 217 ip -netns ioam-tmp-node route add db02::/64 encap ioam6 trace prealloc \ 218 type 0x800000 ns 0 size 4 dev veth0 &>/dev/null 219 tr_ad=$? 220 221 ip -netns ioam-tmp-node -6 route | grep -q "encap ioam6 trace" 222 tr_sh=$? 223 224 if [[ $tr_ad != 0 || $tr_sh != 0 ]] 225 then 226 echo "SKIP: cannot attach an ioam trace to a route, did you compile" \ 227 "without CONFIG_IPV6_IOAM6_LWTUNNEL?" 228 ip link del veth0 2>/dev/null || true 229 ip netns del ioam-tmp-node || true 230 exit 1 231 fi 232 233 ip link del veth0 2>/dev/null || true 234 ip netns del ioam-tmp-node || true 235} 236 237cleanup() 238{ 239 ip link del ioam-veth-alpha 2>/dev/null || true 240 ip link del ioam-veth-gamma 2>/dev/null || true 241 242 ip netns del ioam-node-alpha || true 243 ip netns del ioam-node-beta || true 244 ip netns del ioam-node-gamma || true 245} 246 247setup() 248{ 249 ip netns add ioam-node-alpha 250 ip netns add ioam-node-beta 251 ip netns add ioam-node-gamma 252 253 ip link add name ioam-veth-alpha netns ioam-node-alpha type veth \ 254 peer name ioam-veth-betaL netns ioam-node-beta 255 ip link add name ioam-veth-betaR netns ioam-node-beta type veth \ 256 peer name ioam-veth-gamma netns ioam-node-gamma 257 258 ip -netns ioam-node-alpha link set ioam-veth-alpha name veth0 259 ip -netns ioam-node-beta link set ioam-veth-betaL name veth0 260 ip -netns ioam-node-beta link set ioam-veth-betaR name veth1 261 ip -netns ioam-node-gamma link set ioam-veth-gamma name veth0 262 263 ip -netns ioam-node-alpha addr add db01::2/64 dev veth0 264 ip -netns ioam-node-alpha link set veth0 up 265 ip -netns ioam-node-alpha link set lo up 266 ip -netns ioam-node-alpha route add db02::/64 via db01::1 dev veth0 267 ip -netns ioam-node-alpha route del db01::/64 268 ip -netns ioam-node-alpha route add db01::/64 dev veth0 269 270 ip -netns ioam-node-beta addr add db01::1/64 dev veth0 271 ip -netns ioam-node-beta addr add db02::1/64 dev veth1 272 ip -netns ioam-node-beta link set veth0 up 273 ip -netns ioam-node-beta link set veth1 up 274 ip -netns ioam-node-beta link set lo up 275 276 ip -netns ioam-node-gamma addr add db02::2/64 dev veth0 277 ip -netns ioam-node-gamma link set veth0 up 278 ip -netns ioam-node-gamma link set lo up 279 ip -netns ioam-node-gamma route add db01::/64 via db02::1 dev veth0 280 281 # - IOAM config - 282 ip netns exec ioam-node-alpha sysctl -wq net.ipv6.ioam6_id=${ALPHA[0]} 283 ip netns exec ioam-node-alpha sysctl -wq net.ipv6.ioam6_id_wide=${ALPHA[1]} 284 ip netns exec ioam-node-alpha sysctl -wq net.ipv6.conf.veth0.ioam6_id=${ALPHA[4]} 285 ip netns exec ioam-node-alpha sysctl -wq net.ipv6.conf.veth0.ioam6_id_wide=${ALPHA[5]} 286 ip -netns ioam-node-alpha ioam namespace add 123 data ${ALPHA[6]} wide ${ALPHA[7]} 287 ip -netns ioam-node-alpha ioam schema add ${ALPHA[8]} "${ALPHA[9]}" 288 ip -netns ioam-node-alpha ioam namespace set 123 schema ${ALPHA[8]} 289 290 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.all.forwarding=1 291 ip netns exec ioam-node-beta sysctl -wq net.ipv6.ioam6_id=${BETA[0]} 292 ip netns exec ioam-node-beta sysctl -wq net.ipv6.ioam6_id_wide=${BETA[1]} 293 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth0.ioam6_enabled=1 294 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth0.ioam6_id=${BETA[2]} 295 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth0.ioam6_id_wide=${BETA[3]} 296 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth1.ioam6_id=${BETA[4]} 297 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth1.ioam6_id_wide=${BETA[5]} 298 ip -netns ioam-node-beta ioam namespace add 123 data ${BETA[6]} wide ${BETA[7]} 299 ip -netns ioam-node-beta ioam schema add ${BETA[8]} "${BETA[9]}" 300 ip -netns ioam-node-beta ioam namespace set 123 schema ${BETA[8]} 301 302 ip netns exec ioam-node-gamma sysctl -wq net.ipv6.ioam6_id=${GAMMA[0]} 303 ip netns exec ioam-node-gamma sysctl -wq net.ipv6.ioam6_id_wide=${GAMMA[1]} 304 ip netns exec ioam-node-gamma sysctl -wq net.ipv6.conf.veth0.ioam6_enabled=1 305 ip netns exec ioam-node-gamma sysctl -wq net.ipv6.conf.veth0.ioam6_id=${GAMMA[2]} 306 ip netns exec ioam-node-gamma sysctl -wq net.ipv6.conf.veth0.ioam6_id_wide=${GAMMA[3]} 307 ip -netns ioam-node-gamma ioam namespace add 123 data ${GAMMA[6]} wide ${GAMMA[7]} 308 309 sleep 1 310 311 ip netns exec ioam-node-alpha ping6 -c 5 -W 1 db02::2 &>/dev/null 312 if [ $? != 0 ] 313 then 314 echo "Setup FAILED" 315 cleanup &>/dev/null 316 exit 0 317 fi 318} 319 320log_test_passed() 321{ 322 local desc=$1 323 printf "TEST: %-60s [ OK ]\n" "${desc}" 324} 325 326log_test_failed() 327{ 328 local desc=$1 329 printf "TEST: %-60s [FAIL]\n" "${desc}" 330} 331 332run_test() 333{ 334 local name=$1 335 local desc=$2 336 local node_src=$3 337 local node_dst=$4 338 local ip6_src=$5 339 local ip6_dst=$6 340 local if_dst=$7 341 local trace_type=$8 342 local ioam_ns=$9 343 344 ip netns exec $node_dst ./ioam6_parser $if_dst $name $ip6_src $ip6_dst \ 345 $trace_type $ioam_ns & 346 local spid=$! 347 sleep 0.1 348 349 ip netns exec $node_src ping6 -t 64 -c 1 -W 1 $ip6_dst &>/dev/null 350 if [ $? != 0 ] 351 then 352 log_test_failed "${desc}" 353 kill -2 $spid &>/dev/null 354 else 355 wait $spid 356 [ $? = 0 ] && log_test_passed "${desc}" || log_test_failed "${desc}" 357 fi 358} 359 360run() 361{ 362 echo 363 echo "OUTPUT tests" 364 printf "%0.s-" {1..74} 365 echo 366 367 # set OUTPUT settings 368 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth0.ioam6_enabled=0 369 370 for t in $TESTS_OUTPUT 371 do 372 $t 373 done 374 375 # clean OUTPUT settings 376 ip netns exec ioam-node-beta sysctl -wq net.ipv6.conf.veth0.ioam6_enabled=1 377 ip -netns ioam-node-alpha route change db01::/64 dev veth0 378 379 380 echo 381 echo "INPUT tests" 382 printf "%0.s-" {1..74} 383 echo 384 385 # set INPUT settings 386 ip -netns ioam-node-alpha ioam namespace del 123 387 388 for t in $TESTS_INPUT 389 do 390 $t 391 done 392 393 # clean INPUT settings 394 ip -netns ioam-node-alpha ioam namespace add 123 \ 395 data ${ALPHA[6]} wide ${ALPHA[7]} 396 ip -netns ioam-node-alpha ioam namespace set 123 schema ${ALPHA[8]} 397 ip -netns ioam-node-alpha route change db01::/64 dev veth0 398 399 400 echo 401 echo "GLOBAL tests" 402 printf "%0.s-" {1..74} 403 echo 404 405 for t in $TESTS_GLOBAL 406 do 407 $t 408 done 409} 410 411bit2type=( 412 0x800000 0x400000 0x200000 0x100000 0x080000 0x040000 0x020000 0x010000 413 0x008000 0x004000 0x002000 0x001000 0x000800 0x000400 0x000200 0x000100 414 0x000080 0x000040 0x000020 0x000010 0x000008 0x000004 0x000002 415) 416bit2size=( 4 4 4 4 4 4 4 4 8 8 8 4 4 4 4 4 4 4 4 4 4 4 4 ) 417 418 419################################################################################ 420# # 421# OUTPUT tests # 422# # 423# Two nodes (sender/receiver), IOAM disabled on ingress for the receiver. # 424################################################################################ 425 426out_undef_ns() 427{ 428 ############################################################################## 429 # Make sure that the encap node won't fill the trace if the chosen IOAM # 430 # namespace is not configured locally. # 431 ############################################################################## 432 local desc="Unknown IOAM namespace" 433 434 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 435 type 0x800000 ns 0 size 4 dev veth0 436 437 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 438 db01::1 veth0 0x800000 0 439} 440 441out_no_room() 442{ 443 ############################################################################## 444 # Make sure that the encap node won't fill the trace and will set the # 445 # Overflow flag since there is no room enough for its data. # 446 ############################################################################## 447 local desc="Missing trace room" 448 449 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 450 type 0xc00000 ns 123 size 4 dev veth0 451 452 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 453 db01::1 veth0 0xc00000 123 454} 455 456out_bits() 457{ 458 ############################################################################## 459 # Make sure that, for each trace type bit, the encap node will either: # 460 # (i) fill the trace with its data when it is a supported bit # 461 # (ii) not fill the trace with its data when it is an unsupported bit # 462 ############################################################################## 463 local desc="Trace type with bit <n> only" 464 465 local tmp=${bit2size[22]} 466 bit2size[22]=$(( $tmp + ${#ALPHA[9]} + ((4 - (${#ALPHA[9]} % 4)) % 4) )) 467 468 for i in {0..22} 469 do 470 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace \ 471 prealloc type ${bit2type[$i]} ns 123 size ${bit2size[$i]} \ 472 dev veth0 &>/dev/null 473 474 local cmd_res=$? 475 local descr="${desc/<n>/$i}" 476 477 if [[ $i -ge 12 && $i -le 21 ]] 478 then 479 if [ $cmd_res != 0 ] 480 then 481 npassed=$((npassed+1)) 482 log_test_passed "$descr" 483 else 484 nfailed=$((nfailed+1)) 485 log_test_failed "$descr" 486 fi 487 else 488 run_test "out_bit$i" "$descr" ioam-node-alpha ioam-node-beta \ 489 db01::2 db01::1 veth0 ${bit2type[$i]} 123 490 fi 491 done 492 493 bit2size[22]=$tmp 494} 495 496out_full_supp_trace() 497{ 498 ############################################################################## 499 # Make sure that the encap node will correctly fill a full trace. Be careful,# 500 # "full trace" here does NOT mean all bits (only supported ones). # 501 ############################################################################## 502 local desc="Full supported trace" 503 504 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 505 type 0xfff002 ns 123 size 100 dev veth0 506 507 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 508 db01::1 veth0 0xfff002 123 509} 510 511 512################################################################################ 513# # 514# INPUT tests # 515# # 516# Two nodes (sender/receiver), the sender MUST NOT fill the trace upon # 517# insertion -> the IOAM namespace configured on the sender is removed # 518# and is used in the inserted trace to force the sender not to fill it. # 519################################################################################ 520 521in_undef_ns() 522{ 523 ############################################################################## 524 # Make sure that the receiving node won't fill the trace if the related IOAM # 525 # namespace is not configured locally. # 526 ############################################################################## 527 local desc="Unknown IOAM namespace" 528 529 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 530 type 0x800000 ns 0 size 4 dev veth0 531 532 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 533 db01::1 veth0 0x800000 0 534} 535 536in_no_room() 537{ 538 ############################################################################## 539 # Make sure that the receiving node won't fill the trace and will set the # 540 # Overflow flag if there is no room enough for its data. # 541 ############################################################################## 542 local desc="Missing trace room" 543 544 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 545 type 0xc00000 ns 123 size 4 dev veth0 546 547 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 548 db01::1 veth0 0xc00000 123 549} 550 551in_bits() 552{ 553 ############################################################################## 554 # Make sure that, for each trace type bit, the receiving node will either: # 555 # (i) fill the trace with its data when it is a supported bit # 556 # (ii) not fill the trace with its data when it is an unsupported bit # 557 ############################################################################## 558 local desc="Trace type with bit <n> only" 559 560 local tmp=${bit2size[22]} 561 bit2size[22]=$(( $tmp + ${#BETA[9]} + ((4 - (${#BETA[9]} % 4)) % 4) )) 562 563 for i in {0..11} {22..22} 564 do 565 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace \ 566 prealloc type ${bit2type[$i]} ns 123 size ${bit2size[$i]} dev veth0 567 568 run_test "in_bit$i" "${desc/<n>/$i}" ioam-node-alpha ioam-node-beta \ 569 db01::2 db01::1 veth0 ${bit2type[$i]} 123 570 done 571 572 bit2size[22]=$tmp 573} 574 575in_oflag() 576{ 577 ############################################################################## 578 # Make sure that the receiving node won't fill the trace since the Overflow # 579 # flag is set. # 580 ############################################################################## 581 local desc="Overflow flag is set" 582 583 # Exception: 584 # Here, we need the sender to set the Overflow flag. For that, we will add 585 # back the IOAM namespace that was previously configured on the sender. 586 ip -netns ioam-node-alpha ioam namespace add 123 587 588 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 589 type 0xc00000 ns 123 size 4 dev veth0 590 591 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 592 db01::1 veth0 0xc00000 123 593 594 # And we clean the exception for this test to get things back to normal for 595 # other INPUT tests 596 ip -netns ioam-node-alpha ioam namespace del 123 597} 598 599in_full_supp_trace() 600{ 601 ############################################################################## 602 # Make sure that the receiving node will correctly fill a full trace. Be # 603 # careful, "full trace" here does NOT mean all bits (only supported ones). # 604 ############################################################################## 605 local desc="Full supported trace" 606 607 ip -netns ioam-node-alpha route change db01::/64 encap ioam6 trace prealloc \ 608 type 0xfff002 ns 123 size 80 dev veth0 609 610 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-beta db01::2 \ 611 db01::1 veth0 0xfff002 123 612} 613 614 615################################################################################ 616# # 617# GLOBAL tests # 618# # 619# Three nodes (sender/router/receiver), IOAM fully enabled on every node. # 620################################################################################ 621 622fwd_full_supp_trace() 623{ 624 ############################################################################## 625 # Make sure that all three nodes correctly filled the full supported trace # 626 # by checking that the trace data is consistent with the predefined config. # 627 ############################################################################## 628 local desc="Forward - Full supported trace" 629 630 ip -netns ioam-node-alpha route change db02::/64 encap ioam6 trace prealloc \ 631 type 0xfff002 ns 123 size 244 via db01::1 dev veth0 632 633 run_test ${FUNCNAME[0]} "${desc}" ioam-node-alpha ioam-node-gamma db01::2 \ 634 db02::2 veth0 0xfff002 123 635} 636 637 638################################################################################ 639# # 640# MAIN # 641# # 642################################################################################ 643 644if [ "$(id -u)" -ne 0 ] 645then 646 echo "SKIP: Need root privileges" 647 exit 1 648fi 649 650if [ ! -x "$(command -v ip)" ] 651then 652 echo "SKIP: Could not run test without ip tool" 653 exit 1 654fi 655 656ip ioam &>/dev/null 657if [ $? = 1 ] 658then 659 echo "SKIP: iproute2 too old, missing ioam command" 660 exit 1 661fi 662 663check_kernel_compatibility 664 665cleanup &>/dev/null 666setup 667run 668cleanup &>/dev/null 669