Browse code

Optionally install all pip into a global venv.

This is useful for tracking what pip is causing to be
installed over and beyond os pacakges.

In support of this, move all package installation to before
the section where we install via pip. Leave the deferred configuration
until later though.

Change-Id: I89677fd54635e82b10ab674ddeb9ffb3f1a755f0

Monty Taylor authored on 2012/07/27 01:09:24
Showing 2 changed files
... ...
@@ -471,12 +471,19 @@ function pip_install {
471 471
     if [[ -z "$os_PACKAGE" ]]; then
472 472
         GetOSVersion
473 473
     fi
474
-    if [[ "$os_PACKAGE" = "deb" ]]; then
475
-        CMD_PIP=/usr/bin/pip
474
+    if [[ $TRACK_DEPENDS = True ]] ; then
475
+        source $DEST/.venv/bin/activate
476
+        CMD_PIP=$DEST/.venv/bin/pip
477
+        SUDO_PIP="env"
476 478
     else
477
-        CMD_PIP=/usr/bin/pip-python
479
+        SUDO_PIP="sudo"
480
+        if [[ "$os_PACKAGE" = "deb" ]]; then
481
+            CMD_PIP=/usr/bin/pip
482
+        else
483
+            CMD_PIP=/usr/bin/pip-python
484
+        fi
478 485
     fi
479
-    sudo PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip} \
486
+    $SUDO_PIP PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip} \
480 487
         HTTP_PROXY=$http_proxy \
481 488
         HTTPS_PROXY=$https_proxy \
482 489
         $CMD_PIP install --use-mirrors $@
... ...
@@ -501,12 +508,17 @@ function restart_service() {
501 501
 # develop, so that pip and not distutils process the dependency chain
502 502
 # setup_develop directory
503 503
 function setup_develop() {
504
+    if [[ $TRACK_DEPENDS = True ]] ; then
505
+        SUDO_CMD="env"
506
+    else
507
+        SUDO_CMD="sudo"
508
+    fi
504 509
     (cd $1; \
505 510
         python setup.py egg_info; \
506 511
         raw_links=$(awk '/^.+/ {print "-f " $1}' *.egg-info/dependency_links.txt); \
507 512
         depend_links=$(echo $raw_links | xargs); \
508 513
         pip_install -r *-info/requires.txt $depend_links; \
509
-        sudo \
514
+        $SUDO_CMD \
510 515
             HTTP_PROXY=$http_proxy \
511 516
             HTTPS_PROXY=$https_proxy \
512 517
             python setup.py develop \
... ...
@@ -614,6 +614,130 @@ else
614 614
     install_package $(get_packages $FILES/rpms)
615 615
 fi
616 616
 
617
+if [[ $SYSLOG != "False" ]]; then
618
+    install_package rsyslog-relp
619
+fi
620
+
621
+if is_service_enabled rabbit; then
622
+    # Install rabbitmq-server
623
+    # the temp file is necessary due to LP: #878600
624
+    tfile=$(mktemp)
625
+    install_package rabbitmq-server > "$tfile" 2>&1
626
+    cat "$tfile"
627
+    rm -f "$tfile"
628
+elif is_service_enabled qpid; then
629
+    if [[ "$os_PACKAGE" = "rpm" ]]; then
630
+        install_package qpid-cpp-server
631
+    else
632
+        install_package qpidd
633
+    fi
634
+fi
635
+
636
+if is_service_enabled mysql; then
637
+
638
+    if [[ "$os_PACKAGE" = "deb" ]]; then
639
+        # Seed configuration with mysql password so that apt-get install doesn't
640
+        # prompt us for a password upon install.
641
+        cat <<MYSQL_PRESEED | sudo debconf-set-selections
642
+mysql-server-5.1 mysql-server/root_password password $MYSQL_PASSWORD
643
+mysql-server-5.1 mysql-server/root_password_again password $MYSQL_PASSWORD
644
+mysql-server-5.1 mysql-server/start_on_boot boolean true
645
+MYSQL_PRESEED
646
+    fi
647
+
648
+    # while ``.my.cnf`` is not needed for openstack to function, it is useful
649
+    # as it allows you to access the mysql databases via ``mysql nova`` instead
650
+    # of having to specify the username/password each time.
651
+    if [[ ! -e $HOME/.my.cnf ]]; then
652
+        cat <<EOF >$HOME/.my.cnf
653
+[client]
654
+user=$MYSQL_USER
655
+password=$MYSQL_PASSWORD
656
+host=$MYSQL_HOST
657
+EOF
658
+        chmod 0600 $HOME/.my.cnf
659
+    fi
660
+    # Install mysql-server
661
+    install_package mysql-server
662
+fi
663
+
664
+if is_service_enabled quantum; then
665
+    if [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
666
+        # Install deps
667
+        # FIXME add to files/apts/quantum, but don't install if not needed!
668
+        install_package python-configobj
669
+    fi
670
+fi
671
+
672
+if is_service_enabled horizon; then
673
+    if [[ "$os_PACKAGE" = "deb" ]]; then
674
+        # Install apache2, which is NOPRIME'd
675
+        install_package apache2 libapache2-mod-wsgi
676
+    else
677
+        sudo rm -f /etc/httpd/conf.d/000-*
678
+        install_package httpd mod_wsgi
679
+    fi
680
+fi
681
+
682
+if is_service_enabled q-agt; then
683
+    if [[ "$Q_PLUGIN" = "openvswitch" ]]; then
684
+        # Install deps
685
+        # FIXME add to files/apts/quantum, but don't install if not needed!
686
+        if [[ "$os_PACKAGE" = "deb" ]]; then
687
+            kernel_version=`cat /proc/version | cut -d " " -f3`
688
+            install_package make fakeroot dkms openvswitch-switch openvswitch-datapath-dkms linux-headers-$kernel_version
689
+        else
690
+            ### FIXME(dtroyer): Find RPMs for OpenVSwitch
691
+            echo "OpenVSwitch packages need to be located"
692
+        fi
693
+    elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
694
+       install_package bridge-utils
695
+    fi
696
+fi
697
+
698
+if is_service_enabled n-cpu; then
699
+
700
+    # Virtualization Configuration
701
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
702
+    if [[ "$os_PACKAGE" = "deb" ]]; then
703
+        LIBVIRT_PKG_NAME=libvirt-bin
704
+    else
705
+        LIBVIRT_PKG_NAME=libvirt
706
+    fi
707
+    install_package $LIBVIRT_PKG_NAME
708
+    # Install and configure **LXC** if specified.  LXC is another approach to
709
+    # splitting a system into many smaller parts.  LXC uses cgroups and chroot
710
+    # to simulate multiple systems.
711
+    if [[ "$LIBVIRT_TYPE" == "lxc" ]]; then
712
+        if [[ "$os_PACKAGE" = "deb" ]]; then
713
+            if [[ "$DISTRO" > natty ]]; then
714
+                install_package cgroup-lite
715
+            fi
716
+        else
717
+            ### FIXME(dtroyer): figure this out
718
+            echo "RPM-based cgroup not implemented yet"
719
+            yum_install libcgroup-tools
720
+        fi
721
+    fi
722
+fi
723
+
724
+if is_service_enabled swift; then
725
+    # Install memcached for swift.
726
+    install_package memcached
727
+fi
728
+
729
+TRACK_DEPENDS=${TRACK_DEPENDS:-False}
730
+
731
+# Install python packages into a virtualenv so that we can track them
732
+if [[ $TRACK_DEPENDS = True ]] ; then
733
+    install_package python-virtualenv
734
+
735
+    rm -rf $DEST/.venv
736
+    virtualenv --system-site-packages $DEST/.venv
737
+    source $DEST/.venv/bin/activate
738
+    $DEST/.venv/bin/pip freeze > $DEST/requires-pre-pip
739
+fi
740
+
617 741
 # Install python requirements
618 742
 pip_install $(get_packages $FILES/pips | sort -u)
619 743
 
... ...
@@ -671,7 +795,6 @@ if is_service_enabled cinder; then
671 671
     install_cinder
672 672
 fi
673 673
 
674
-
675 674
 # Initialization
676 675
 # ==============
677 676
 
... ...
@@ -715,12 +838,19 @@ if is_service_enabled cinder; then
715 715
     configure_cinder
716 716
 fi
717 717
 
718
+if [[ $TRACK_DEPENDS = True ]] ; then
719
+    $DEST/.venv/bin/pip freeze > $DEST/requires-post-pip
720
+    if ! diff -Nru $DEST/requires-pre-pip $DEST/requires-post-pip > $DEST/requires.diff ; then
721
+        cat $DEST/requires.diff
722
+    fi
723
+    echo "Ran stack.sh in depend tracking mode, bailing out now"
724
+    exit 0
725
+fi
718 726
 
719 727
 # Syslog
720 728
 # ------
721 729
 
722 730
 if [[ $SYSLOG != "False" ]]; then
723
-    install_package rsyslog-relp
724 731
     if [[ "$SYSLOG_HOST" = "$HOST_IP" ]]; then
725 732
         # Configure the master host to receive
726 733
         cat <<EOF >/tmp/90-stack-m.conf
... ...
@@ -743,12 +873,7 @@ fi
743 743
 # --------------
744 744
 
745 745
 if is_service_enabled rabbit; then
746
-    # Install and start rabbitmq-server
747
-    # the temp file is necessary due to LP: #878600
748
-    tfile=$(mktemp)
749
-    install_package rabbitmq-server > "$tfile" 2>&1
750
-    cat "$tfile"
751
-    rm -f "$tfile"
746
+    # Start rabbitmq-server
752 747
     if [[ "$os_PACKAGE" = "rpm" ]]; then
753 748
         # RPM doesn't start the service
754 749
         restart_service rabbitmq-server
... ...
@@ -756,45 +881,17 @@ if is_service_enabled rabbit; then
756 756
     # change the rabbit password since the default is "guest"
757 757
     sudo rabbitmqctl change_password guest $RABBIT_PASSWORD
758 758
 elif is_service_enabled qpid; then
759
-    if [[ "$os_PACKAGE" = "rpm" ]]; then
760
-        install_package qpid-cpp-server
761
-        restart_service qpidd
762
-    else
763
-        install_package qpidd
764
-    fi
759
+    restart_service qpidd
765 760
 fi
766 761
 
767 762
 
768 763
 # Mysql
769 764
 # -----
770 765
 
771
-if is_service_enabled mysql; then
772 766
 
773
-    if [[ "$os_PACKAGE" = "deb" ]]; then
774
-        # Seed configuration with mysql password so that apt-get install doesn't
775
-        # prompt us for a password upon install.
776
-        cat <<MYSQL_PRESEED | sudo debconf-set-selections
777
-mysql-server-5.1 mysql-server/root_password password $MYSQL_PASSWORD
778
-mysql-server-5.1 mysql-server/root_password_again password $MYSQL_PASSWORD
779
-mysql-server-5.1 mysql-server/start_on_boot boolean true
780
-MYSQL_PRESEED
781
-    fi
782
-
783
-    # while ``.my.cnf`` is not needed for openstack to function, it is useful
784
-    # as it allows you to access the mysql databases via ``mysql nova`` instead
785
-    # of having to specify the username/password each time.
786
-    if [[ ! -e $HOME/.my.cnf ]]; then
787
-        cat <<EOF >$HOME/.my.cnf
788
-[client]
789
-user=$MYSQL_USER
790
-password=$MYSQL_PASSWORD
791
-host=$MYSQL_HOST
792
-EOF
793
-        chmod 0600 $HOME/.my.cnf
794
-    fi
767
+if is_service_enabled mysql; then
795 768
 
796
-    # Install and start mysql-server
797
-    install_package mysql-server
769
+    #start mysql-server
798 770
     if [[ "$os_PACKAGE" = "rpm" ]]; then
799 771
         # RPM doesn't start the service
800 772
         start_service mysqld
... ...
@@ -904,10 +1001,8 @@ if is_service_enabled horizon; then
904 904
     sudo mkdir -p $HORIZON_DIR/.blackhole
905 905
 
906 906
     if [[ "$os_PACKAGE" = "deb" ]]; then
907
-        # Install apache2, which is NOPRIME'd
908 907
         APACHE_NAME=apache2
909 908
         APACHE_CONF=sites-available/horizon
910
-        install_package apache2 libapache2-mod-wsgi
911 909
         # Clean up the old config name
912 910
         sudo rm -f /etc/apache2/sites-enabled/000-default
913 911
         # Be a good citizen and use the distro tools here
... ...
@@ -917,8 +1012,6 @@ if is_service_enabled horizon; then
917 917
         # Install httpd, which is NOPRIME'd
918 918
         APACHE_NAME=httpd
919 919
         APACHE_CONF=conf.d/horizon.conf
920
-        sudo rm -f /etc/httpd/conf.d/000-*
921
-        install_package httpd mod_wsgi
922 920
         sudo sed '/^Listen/s/^.*$/Listen 0.0.0.0:80/' -i /etc/httpd/conf/httpd.conf
923 921
     fi
924 922
     ## Configure apache to run horizon
... ...
@@ -1028,9 +1121,6 @@ if is_service_enabled quantum; then
1028 1028
             Q_PLUGIN_CLASS="quantum.plugins.openvswitch.ovs_quantum_plugin.OVSQuantumPluginV2"
1029 1029
         fi
1030 1030
     elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1031
-        # Install deps
1032
-        # FIXME add to files/apts/quantum, but don't install if not needed!
1033
-        install_package python-configobj
1034 1031
         Q_PLUGIN_CONF_PATH=etc/quantum/plugins/linuxbridge
1035 1032
         Q_PLUGIN_CONF_FILENAME=linuxbridge_conf.ini
1036 1033
         Q_DB_NAME="quantum_linux_bridge"
... ...
@@ -1104,15 +1194,6 @@ fi
1104 1104
 # Quantum agent (for compute nodes)
1105 1105
 if is_service_enabled q-agt; then
1106 1106
     if [[ "$Q_PLUGIN" = "openvswitch" ]]; then
1107
-        # Install deps
1108
-        # FIXME add to files/apts/quantum, but don't install if not needed!
1109
-        if [[ "$os_PACKAGE" = "deb" ]]; then
1110
-            kernel_version=`cat /proc/version | cut -d " " -f3`
1111
-            install_package make fakeroot dkms openvswitch-switch openvswitch-datapath-dkms linux-headers-$kernel_version
1112
-        else
1113
-            ### FIXME(dtroyer): Find RPMs for OpenVSwitch
1114
-            echo "OpenVSwitch packages need to be located"
1115
-        fi
1116 1107
         # Set up integration bridge
1117 1108
         OVS_BRIDGE=${OVS_BRIDGE:-br-int}
1118 1109
         for PORT in `sudo ovs-vsctl --no-wait list-ports $OVS_BRIDGE`; do
... ...
@@ -1126,8 +1207,7 @@ if is_service_enabled q-agt; then
1126 1126
         AGENT_BINARY="$QUANTUM_DIR/quantum/plugins/openvswitch/agent/ovs_quantum_agent.py"
1127 1127
     elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1128 1128
        # Start up the quantum <-> linuxbridge agent
1129
-       install_package bridge-utils
1130
-        #set the default network interface
1129
+       # set the default network interface
1131 1130
        QUANTUM_LB_PRIVATE_INTERFACE=${QUANTUM_LB_PRIVATE_INTERFACE:-$GUEST_INTERFACE_DEFAULT}
1132 1131
        sudo sed -i -e "s/^physical_interface = .*$/physical_interface = $QUANTUM_LB_PRIVATE_INTERFACE/g" /$Q_PLUGIN_CONF_FILE
1133 1132
        AGENT_BINARY="$QUANTUM_DIR/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py"
... ...
@@ -1273,15 +1353,6 @@ function clean_iptables() {
1273 1273
 
1274 1274
 if is_service_enabled n-cpu; then
1275 1275
 
1276
-    # Virtualization Configuration
1277
-    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1278
-    if [[ "$os_PACKAGE" = "deb" ]]; then
1279
-        LIBVIRT_PKG_NAME=libvirt-bin
1280
-    else
1281
-        LIBVIRT_PKG_NAME=libvirt
1282
-    fi
1283
-    install_package $LIBVIRT_PKG_NAME
1284
-
1285 1276
     # Force IP forwarding on, just on case
1286 1277
     sudo sysctl -w net.ipv4.ip_forward=1
1287 1278
 
... ...
@@ -1304,9 +1375,7 @@ if is_service_enabled n-cpu; then
1304 1304
     # to simulate multiple systems.
1305 1305
     if [[ "$LIBVIRT_TYPE" == "lxc" ]]; then
1306 1306
         if [[ "$os_PACKAGE" = "deb" ]]; then
1307
-            if [[ "$DISTRO" > natty ]]; then
1308
-                install_package cgroup-lite
1309
-            else
1307
+            if [[ ! "$DISTRO" > natty ]]; then
1310 1308
                 cgline="none /cgroup cgroup cpuacct,memory,devices,cpu,freezer,blkio 0 0"
1311 1309
                 sudo mkdir -p /cgroup
1312 1310
                 if ! grep -q cgroup /etc/fstab; then
... ...
@@ -1316,10 +1385,6 @@ if is_service_enabled n-cpu; then
1316 1316
                     sudo mount /cgroup
1317 1317
                 fi
1318 1318
             fi
1319
-        else
1320
-            ### FIXME(dtroyer): figure this out
1321
-            echo "RPM-based cgroup not implemented yet"
1322
-            yum_install libcgroup-tools
1323 1319
         fi
1324 1320
     fi
1325 1321
 
... ...
@@ -1414,8 +1479,6 @@ fi
1414 1414
 # ---------------
1415 1415
 
1416 1416
 if is_service_enabled swift; then
1417
-    # Install memcached for swift.
1418
-    install_package memcached
1419 1417
 
1420 1418
     # We make sure to kill all swift processes first
1421 1419
     swift-init all stop || true