noursaidi · noursaidi · Jun 24, 2020 · Jun 25, 2020 · Jun 25, 2020 · Jun 25, 2020
diff --git a/.checkstyle.xml b/.checkstyle.xml
@@ -42,7 +42,7 @@
 
     <module name="LineLength">
         <property name="fileExtensions" value="java"/>
-        <property name="max" value="100"/>
+        <property name="max" value="120"/>
         <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
     </module>
 

diff --git a/.gitignore b/.gitignore
@@ -21,16 +21,14 @@ validations/
 *.save
 
 # Runtime or sub-module files
-inst/
-faucet/
-forch/
-mininet/
-local/
-local_xxx
-local.old
-firebase/.firebaserc
-firebase/.firebase
-firebase/functions/package-lock.json
+/inst/
+/faucet/
+/forch/
+/mininet/
+/local/
+/firebase/.firebaserc
+/firebase/.firebase
+/firebase/functions/package-lock.json
 nohup.out
 **/node_modules/
 .vscode/

diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/bin/alt_faucet b/bin/alt_faucet
@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+cd $ROOT
+source etc/config_base.sh
+
+if [ -z "$switch_setup_ext_br" ]; then
+   echo switch_setup.ext_br not defined for alternate faucet setup.
+   false
+fi
+
+if [ -z "$switch_setup_alt_port" ]; then
+   echo switch_setup.alt_port not defined for alternate faucet setup.
+   false
+fi
+
+inst_name=$switch_setup_ext_br
+
+inst_dir=inst/faucet/daq-faucet-$inst_name
+mkdir -p $inst_dir
+cp config/faucet/faucet_$inst_name.yaml $inst_dir/faucet.yaml
+echo Launching alternate faucet install $inst_name on $switch_setup_alt_port
+echo DAQ autoclean docker kill daq-faucet-$inst_name
+cmd/faucet $inst_name $switch_setup_alt_port
diff --git a/bin/build_proto b/bin/build_proto
@@ -32,6 +32,11 @@ sha1sum $proto_files > $WEB_ROOT/protos.hash
 
 gen_path=$ROOT/protoc-gen-doc/bin/protoc-gen-doc
 
+if [ -d venv ]; then
+    echo Entering virtual python environment...
+    source venv/bin/activate
+fi
+
 mkdir -p build/daq/proto build/proto
 cp $proto_files build/daq/proto/
 proto_files2=
@@ -56,3 +61,5 @@ mkdir -p libs/proto/
 touch libs/proto/__init__.py
 cp build/daq/proto/*.py libs/proto/
 cp build/protos.html $WEB_ROOT/
+
+python3 -m grpc_tools.protoc -I usi/src/main/proto/ --python_out=libs/proto/ --grpc_python_out=libs/proto/ usi/src/main/proto/usi.proto
diff --git a/bin/external_ovs b/bin/external_ovs
@@ -6,14 +6,19 @@ source etc/config_base.sh
 
 ext_intf=$switch_setup_data_intf
 ext_dpid=$switch_setup_of_dpid
-ext_ofpt=$switch_setup_lo_port
 ext_brid=$switch_setup_ext_br
 ext_brpt=$switch_setup_uplink_port
 
 ext_pri=${ext_intf}
 ext_sec=${ext_intf%-pri}-sec
 
-echo ext_dpid is $ext_dpid
+if [ -z "$switch_setup_alt_port" ]; then
+    ext_ofpt=$switch_setup_lo_port
+else
+    ext_ofpt=$switch_setup_alt_port
+fi
+
+echo ext_dpid is $ext_dpid on port $ext_ofpt
 echo network_config is $network_config
 
 dpid=$(printf %016x $ext_dpid)

diff --git a/bin/registrar b/bin/registrar
@@ -27,4 +27,4 @@ validator/bin/build > /dev/null
 
 echo Running tools version `git describe`
 
-validator/bin/registrar $project_id $site_path $schema_path $*
+validator/bin/registrar $project_id $site_path $schema_path $* 2>&1
diff --git a/bin/setup_dev b/bin/setup_dev
@@ -75,10 +75,6 @@ $AG install \
     python$PVERSION python3-pkg-resources python3-setuptools \
     python$PVERSION-dev python3-pip python emacs-nox python$PVERSION-venv
 
-# Jump through some hoops for mininet, which still has some python2 deps.
-$AG install python-pip
-python2 -m pip install setuptools
-
 if [ -d mininet ]; then
     echo Checking mininet version matches $MININETV...
     targetrev=$(cd mininet; git rev-parse $MININETV)
@@ -137,8 +133,9 @@ $PIP install --upgrade --index-url=https://pypi.python.org/simple Jinja2 \
     google-api-core==1.16.0 \
     google-cloud-storage==1.16.1 \
     google-cloud-firestore==1.6.0 \
-    google-cloud-logging==1.14.0
-
+    google-cloud-logging==1.14.0 \
+    grpcio-tools==1.30.0
+
 $PIP freeze
 echo Resetting .cache directory permissions...
 test -n "$USER" && sudo chown $USER -R $HOME/.cache

diff --git a/bin/setup_testing b/bin/setup_testing
diff --git a/bin/test_daq b/bin/test_daq
@@ -31,6 +31,16 @@ echo -n "DAQ version "
 git describe --dirty --always
 echo
 
+TAGGED_VERSION=`cat etc/docker_images.ver`
+if ! git show $TAGGED_VERSION > /dev/null; then
+    echo
+    echo Tagged version $TAGGED_VERSION not found.
+    echo Maybe you need to fetch tags: git fetch --tags.
+    echo If this is on Travis, ensure tags were pushed to your repo.
+    echo
+    false
+fi
+
 if [ -d faucet ]; then
     echo -n "Last FAUCET commit "
     (cd $FAUCET; git log -n 1 --pretty=format:"%h - %an, %ar : %s" || true)

diff --git a/bin/troubleshoot b/bin/troubleshoot
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+ROOT=$(realpath $(dirname $0)/..)
+cd $ROOT
+
+if [ ! -d inst ]; then
+    echo "Error: run this script after a test run completes"
+    exit 1
+fi
+
+# After the system settles (early on some dpid=1 messages are expected) if we see 
+# unknown dpid in faucet log, dpid might be misconfigured
+unknown_dpid=`fgrep 'unknown datapath' inst/faucet.log | wc -l`
+if [ "$unknown_dpid" -gt 20 ]; then
+    echo "Error: Faucet reports unknown datapath DPID:"
+    fgrep 'unknown datapath' inst/faucet.log | tail -n1
+    echo "Check if switch_setup:of_dpid in config matches the DPID on the physical switch"
+else
+    echo "Checking DPID misconfig: ok"
+fi
+
+# If the switch test failed with a monitoring timeout, switch login info could be wrong
+switch_timeout=`fgrep 'Monitoring timeout for switch' inst/cmdrun.log`
+if [ -n "$switch_timeout" ]; then
+    echo "Error: Timeout connecting to physical switch"
+    echo "Check switch username/password configuration"
+else
+    echo "Checking Switch timeout: ok"
+fi
diff --git a/bin/validate b/bin/validate
@@ -20,16 +20,14 @@ if [ -z "$schema_path" ]; then
   false
 fi
 
-validator/bin/build
+echo Building validator...
+validator/bin/build > /dev/null
 
-unset GOOGLE_CLOUD_PROJECT
-export GOOGLE_APPLICATION_CREDENTIALS=$PWD/$gcp_cred
-echo Using credentials from $GOOGLE_APPLICATION_CREDENTIALS
 echo Configured topic is $gcp_topic
 echo Configured schema is $schema_path
-if [ -n "$site_path" ]; then
-    echo Configured site path is $site_path
-fi
+echo Configured site path is $site_path
 echo
 
+echo Running tools version `git describe`
+
 validator/bin/validate $schema_path pubsub:$gcp_topic dev $site_path
diff --git a/cmd/build b/cmd/build
@@ -26,11 +26,12 @@ DOCKER_IMAGE_VER=docker_images.ver
 
 cd $ROOT
 source etc/config_base.sh
-host_tests=$host_tests bin/docker_build_files
+echo host_tests=$host_tests
+test_targets=$(host_tests=$host_tests bin/docker_build_files)
 function pull_images {
     TAG=$1
     declare -A test_set
-    for target in $(host_tests=$host_tests bin/docker_build_files); do
+    for target in $test_targets; do
         target=$(echo $target | sed 's|^.*/Dockerfile.||' | echo daqf/$(</dev/stdin))
         test_set[$target]=y
     done

diff --git a/cmd/exrun b/cmd/exrun
@@ -44,6 +44,7 @@ FAUCET=$(realpath $ROOT/faucet)
 FORCH=$(realpath $ROOT/forch)
 MININET=$(realpath $ROOT/mininet)
 LIBS=$(realpath $ROOT/libs)
+PROTO=$(realpath $ROOT/libs/proto)
 
 if [ ! -d $FAUCET ]; then
     echo Faucet directory $FAUCET not found, try running bin/setup_dev.
@@ -66,10 +67,10 @@ if [ "$1" == "-h" ]; then
     skip_autostart=y
 fi
 
-export PYTHONPATH=$FORCH:$FAUCET:$MININET:$LIBS
+export PYTHONPATH=$FORCH:$FAUCET:$MININET:$LIBS:$PROTO
 
 mkdir -p $INSTDIR
-rm -f $INSTDIR/faucet* $cleanup_file
+rm -rf $INSTDIR/faucet* $cleanup_file
 
 docker ps > /dev/null 2>&1 || service docker start
 
@@ -81,7 +82,8 @@ sudo rm -f $cleanup_file
 function autostart {
     tmp=`mktemp`
     echo DAQ autostart $@
-    eval $@ | tee $tmp
+    eval $@ > $tmp # Don't use "eval $@ | tee $tmp" here; breaks cmd/usi.
+    cat $tmp
     grep -e '^\s*DAQ autoclean\s' $tmp >> $cleanup_file || true
 }
 
@@ -99,12 +101,19 @@ if [ -n "$switch_setup_ext_br" ]; then
     autostart bin/external_ovs
 fi
 
+if [ -n "$switch_setup_alt_port" ]; then
+    autostart bin/alt_faucet
+fi
+
 if [ -n "$switch_setup_model" ]; then
     autostart bin/physical_sec
 else
     echo No external switch model specified.
 fi
 
+docker rm -f daq-usi || true
+autostart cmd/usi
+
 # Kill any gateways so that they don't prematurely assign an IP address.
 gwids=$(docker ps --format '{{ .Image }} {{ .Names }}' | fgrep daqf/networking | awk '{print $2}') || true
 for gwid in $gwids; do

diff --git a/cmd/usi b/cmd/usi
@@ -0,0 +1,18 @@
+#!/bin/bash -e
+TMP_DIR=/tmp/usi
+
+function dump_ovs_interfaces {
+    while true; do
+        sudo ovs-ofctl show sec > $TMP_DIR/ovs_output.txt || true
+        sleep 5
+    done
+}
+
+echo Starting USI
+mkdir -p $TMP_DIR
+dump_ovs_interfaces &
+PID=$!
+docker run -d -v /tmp/usi:/ovs --privileged --network=host --name daq-usi daqf/usi
+
+echo DAQ autoclean docker kill daq-usi 
+echo DAQ autoclean kill $PID 
diff --git a/config/faucet/faucet_alt-switch.yaml b/config/faucet/faucet_alt-switch.yaml
@@ -0,0 +1,22 @@
+dps:
+  alt-switch:
+    dp_id: 2
+    interfaces:
+      1:
+        native_vlan: 1001
+      2:
+        native_vlan: 1002
+      3:
+        native_vlan: 1003
+      4:
+        native_vlan: 1004
+      5:
+        native_vlan: 1005
+      100:
+        tagged_vlans: [1001, 1002, 1003, 1004, 1005]
+vlans:
+  1001:
+  1002:
+  1003:
+  1004:
+  1005:
diff --git a/config/modules/all.conf b/config/modules/all.conf
@@ -7,4 +7,5 @@ include subset/connection/build.conf
 include subset/bacnet/build.conf
 include subset/security/build.conf
 include subset/cloud/build.conf
-include subset/manual/build.conf
+include subset/manual/build.conf
+include subset/ntp/build.conf
Original file line number	Diff line number	Diff line change
Expand Up		@@ -27,4 +27,4 @@ validator/bin/build > /dev/null

		echo Running tools version `git describe`

		validator/bin/registrar $project_id $site_path $schema_path $*
		validator/bin/registrar $project_id $site_path $schema_path $* 2>&1