HCF-451 Fix deployment hooks for proxy configuration

Also set up a AWS terraform configuration to use proxies.

.gitignore

@@ -13,7 +13,8 @@ password
 .vagrant
 *.box
 *.vmdk
-hcf-ucp.json
+/hcf-ucp.json
 .demos
-hcf.tf
 src/*-clone
+/*.tf
+/*.tf.json

Makefile

@@ -270,6 +270,16 @@ aws:
 	  "RBENV_VERSION=2.2.3 ${CURDIR}/bin/rm-transformer.rb ${DTR} ${ENV_DIR_MAKE} --provider tf:aws ${CURDIR}/container-host-files/etc/hcf/config/role-manifest.yml" > "${CURDIR}/hcf-aws.tf.json" ; \
 	echo Generated ${CURDIR}/hcf-aws.tf.json
 
+aws-proxy:
+	$(call print_status, Generate AWS terraform configuration with proxy)
+	cp terraform/aws-proxy.tf ${CURDIR}
+	@docker run --rm \
+	  -v ${CURDIR}:${CURDIR} \
+	  helioncf/hcf-pipeline-ruby-bosh \
+	  bash -l -c \
+	  "RBENV_VERSION=2.2.3 ${CURDIR}/bin/rm-transformer.rb ${DTR} ${ENV_DIR_MAKE} --provider tf:aws:proxy ${CURDIR}/container-host-files/etc/hcf/config/role-manifest.yml" > "${CURDIR}/hcf-aws-proxy.tf.json" ; \
+	echo Generated ${CURDIR}/hcf-aws-proxy.tf.json
+
 ########## DISTRIBUTION TARGETS ##########
 
 dist: mpc-dist aws-dist

bin/dev-settings.env

@@ -78,3 +78,13 @@ RESOURCE_DIRECTORY_KEY=cc-resources
 DEVSVC_MONGODB_ADMIN_PASSWORD=mongo_password
 DEVSVC_MYSQL_ADMIN_PASSWORD=mysql_password
 DEVSVC_POSTGRES_ADMIN_PASSWORD=postgres_password
+DNS_SUFFIX=foobar
+SKIP_CERT_VERIFY=true
+TRUSTED_CERTS=
+
+# HTTP_PROXY, HTTPS_PROXY, NO_PROXY are set via the transfomer where applicable
+HTTP_PROXY=
+http_proxy=
+HTTPS_PROXY=
+https_proxy=
+NO_PROXY=.hcf,127.0.0.1

bin/rm-transformer.rb

@@ -104,6 +104,10 @@ def provider_constructor
       require_relative 'rm-transformer/tf-aws'
       ToTerraformAWS
     },
+    'tf:aws:proxy' => lambda {
+      require_relative 'rm-transformer/tf-aws-proxy'
+      ToTerraformAWSWithProxy
+    },
     'tf:mpc' => lambda {
       require_relative 'rm-transformer/tf-mpc'
       ToTerraformMPC
@@ -123,7 +127,8 @@ def load_role_manifest(path, env_dir)
     vars = role_manifest['configuration']['variables']
     Dir.glob(File.join(env_dir, "*.env")).sort.each do |env_file|
       File.readlines(env_file).each do |line|
-        name, value = line.chomp.split('=', 2)
+        next if /^($|\s*#)/ =~ line  # Skip empty lines and comments
+        name, value = line.strip.split('=', 2)
         i = vars.find_index{|x| x['name'] == name }
         if i.nil?
           STDERR.puts "Variable #{name} defined in #{env_file} does not exist in role manifest"

bin/rm-transformer/tf-aws-proxy.rb

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+## Terraform/AWS with proxy output provider
+## Specialization of generic TF to test a proxied configuration
+# # ## ### ##### ########
+
+require_relative 'tf-aws'
+
+class ToTerraformAWSWithProxy < ToTerraformAWS
+
+  def special_variables
+    super + %w(HTTP_PROXY http_proxy HTTPS_PROXY https_proxy)
+  end
+
+  def security_groups
+    super.merge(
+      'backend' => {
+        'tags' => {
+          'Name' => '${var.cluster-prefix}-backend'
+        },
+        'name' => '${var.cluster-prefix}-backend',
+        'description' => 'Backend',
+        'vpc_id' => '${aws_vpc.cluster.id}'
+      }
+    )
+  end
+
+  def security_group_rules(roles)
+    rules = {}
+    rules.merge! frontend_security_group_rules
+    rules.merge! exposed_ports_security_group_rules('backend', roles)
+    rules.merge! backend_security_group_rules
+    rules
+  end
+
+  def frontend_security_group_rules
+    super.merge(
+      'frontend_ingress_ssh' => {
+        # Allow proxy inbound traffic to sshd
+        'security_group_id' => '${aws_security_group.frontend.id}',
+        'type' => 'ingress',
+        'from_port' => 22,
+        'to_port' => 22,
+        'protocol' => 'tcp',
+        'cidr_blocks' => ['0.0.0.0/0']
+      },
+      'frontend_ingress_http_proxy' => {
+        # Allow proxy inbound traffic to sshd
+        'security_group_id' => '${aws_security_group.frontend.id}',
+        'type' => 'ingress',
+        'from_port' => 3128,
+        'to_port' => 3128,
+        'protocol' => 'tcp',
+        'source_security_group_id' => '${aws_security_group.backend.id}'
+      }
+    )
+  end
+
+  def backend_security_group_rules
+    {
+      'backend_egress_self' => {
+        # Allow core outbound traffic to itself
+        'security_group_id' => '${aws_security_group.backend.id}',
+        'type' => 'egress',
+        'from_port' => 0,
+        'to_port' => 0,
+        'protocol' => '-1',
+        'self' => true
+      },
+      'backend_egress_frontend' => {
+        # Allow core outbound traffic to the proxy
+        'security_group_id' => '${aws_security_group.backend.id}',
+        'type' => 'egress',
+        'from_port' => 3128,
+        'to_port' => 3128,
+        'protocol' => 'tcp',
+        'source_security_group_id' => '${aws_security_group.frontend.id}'
+      },
+      'backend_egress_hairpin' => {
+        # Allow core outbound traffic to itself via the public IP
+        'security_group_id' => '${aws_security_group.backend.id}',
+        'type' => 'egress',
+        'from_port' => 0,
+        'to_port' => 0,
+        'protocol' => '-1',
+        'cidr_blocks' => ['${null_resource.PUBLIC_IP.triggers.PUBLIC_IP}/32']
+      }
+    }
+  end
+end

bin/rm-transformer/tf-aws.rb

@@ -11,46 +11,74 @@ class ToTerraformAWS < ToTerraform
   # Internal definitions
 
   def to_terraform(manifest)
-    emit_security_group(manifest['roles'])
+    @out['resource'] ||= []
+    # To avoid cyclic dependencies, we use separate security group rules
+    # instead of embedding them in the security group definitions
+    @out['resource'] << { 'aws_security_group' => security_groups }
+    @out['resource'] << {
+      'aws_security_group_rule' => security_group_rules(manifest['roles'])
+    }
     super(manifest)
   end
 
-  def emit_security_group(roles)
-    rules = get_exposed_ports(roles).map do |port|
-      ({
-        'from_port' => port['target'],
-        'to_port' =>   port['target'],
-        'protocol' => port['protocol'].downcase,
-        'cidr_blocks' => ['0.0.0.0/0']
-      })
-    end
+  def security_groups
+    {
+      'frontend' => {
+        'tags' => {
+          'Name' => '${var.cluster-prefix}-frontend'
+        },
+        'name' => '${var.cluster-prefix}-frontend',
+        'description' => 'Frontend',
+        'vpc_id' => '${aws_vpc.cluster.id}'
+      }
+    }
+  end
 
-    @out['resource'] ||= []
-    @out['resource'] << {
-      # Add a security group for the Frontend endpoints
-      'aws_security_group' => {
-        'frontend' => {
-          'tags' => {
-            'Name' => '${var.cluster-prefix}-frontend'
-          },
-          'name' => '${var.cluster-prefix}-frontend',
-          'description' => 'Frontend',
-          'vpc_id' => '${aws_vpc.cluster.id}',
-          'egress' => [
-            # Allow outbound traffic to all ports everywhere
-            {
-              'from_port' => 0,
-              'to_port' => 0,
-              'protocol' => '-1',
-              'cidr_blocks' => ['0.0.0.0/0']
-            }
-          ],
-          'ingress' => rules
-        }
+  def security_group_rules(roles)
+    rules = {}
+    rules.merge! frontend_security_group_rules
+    rules.merge! exposed_ports_security_group_rules('frontend', roles)
+    rules
+  end
+
+  # Get the security group rules for the frontend machine
+  # This may be a proxy in some configurations
+  def frontend_security_group_rules
+    {
+      'frontend_egress' => {
+        # Allow outbound traffic to all ports everywhere
+        'security_group_id' => '${aws_security_group.frontend.id}',
+        'type' => 'egress',
+        'from_port' => 0,
+        'to_port' => 0,
+        'protocol' => '-1',
+        'cidr_blocks' => ['0.0.0.0/0']
       }
     }
   end
 
+  # Get the security group rules for exposed ports
+  #
+  # @param group_name [String] The name of the security group
+  # @param roles      [Hash]   The roles definition
+  # @returns          [Hash]   The exposed ports security group rules
+  def exposed_ports_security_group_rules(group_name, roles)
+    rules = {}
+    get_exposed_ports(roles).each do |port|
+      protocol = port['protocol'].downcase
+      target = port['target'].to_i
+      rules["#{group_name}_ingress_#{protocol}_to_#{target}"] = {
+        'security_group_id' => "${aws_security_group.#{group_name}.id}",
+        'type' => 'ingress',
+        'from_port' => target,
+        'to_port' => target,
+        'protocol' => protocol,
+        'cidr_blocks' => ['0.0.0.0/0']
+      }
+    end
+    rules
+  end
+
   # # ## ### ##### ########
 end
 

bin/rm-transformer/tf.rb

@@ -9,8 +9,7 @@
 class ToTerraform < Common
   def initialize(options)
     super(options)
-    @have_public_ip = false
-    @have_domain = false
+    @have_specials = []
     @out = {}
   end
 
@@ -73,28 +72,28 @@ def emit_null(name, value)
   def emit_configuration(manifest)
     manifest['configuration']['variables'].each do |config|
       name = config['name']
-      next if special?(name)
+      if special_variables.include?(name)
+        @have_specials << name
+        next
+      end
       value = config['default']
       # Ignore optional values without a default.
       next if value.nil? && !config['required']
       emit_variable(name, value: value)
     end
-    puts 'PUBLIC_IP is missing from input role-manifest' unless @have_public_ip
-    puts 'DOMAIN is missing from input role-manifest' unless @have_domain
+    missing = special_variables.sort - @have_specials.sort
+    missing.each do |var_name|
+      STDERR.puts "#{var_name} is missing from input role-manifest"
+    end
   end
 
-  def special?(name)
+  def special_variables
     # Special case two RM variables (null_resource). We skip them
     # and expect an external file to provide a null_resource wiring
     # them to the networking setup. Because only the external
     # context knows where the value actually comes from in terms of
     # vars, etc.
-    case name
-    when 'PUBLIC_IP' then @have_public_ip = true
-    when 'DOMAIN'    then @have_domain = true
-    else return false
-    end
-    true
+    %w(PUBLIC_IP DOMAIN)
   end
 
   def emit_dtr_variables
@@ -157,9 +156,7 @@ def emit_settings(manifest)
   def make_assignment_for(name)
     # Note, no double-quotes around any values. Would become part of
     # the value when docker run read the --env-file. Bad.
-    if name == 'PUBLIC_IP'
-      %(#{name}=\$\{null_resource.#{name}.triggers.#{name}\}\n)
-    elsif name == 'DOMAIN'
+    if special_variables.include? name
       %(#{name}=\$\{null_resource.#{name}.triggers.#{name}\}\n)
     else
       %(#{name}=\$\{replace(var.#{name},"\\\\n", "\\\\\\\\n")\}\n)

container-host-files/etc/hcf/config/role-manifest.yml

@@ -1163,6 +1163,19 @@ configuration:
     secret: true
   - name: DEVSVC_POSTGRES_ADMIN_PASSWORD
     secret: true
+  - name: SKIP_CERT_VERIFY
+    default: true
+  # Proxy
+  - name: HTTP_PROXY
+    default: ""
+  - name: http_proxy
+    default: ""
+  - name: HTTPS_PROXY
+    default: ""
+  - name: https_proxy
+    default: ""
+  - name: NO_PROXY
+    default: ""
   templates:
     index: 0
     networks.default.ip: '"((IP_ADDRESS))"'
@@ -1238,6 +1251,7 @@ configuration:
     properties.uaa.clients.tcp_router.secret: '"((UAA_CLIENTS_TCP_ROUTER_SECRET))"'
     properties.uaa.scim.users: '["((CLUSTER_ADMIN_USERNAME))|((CLUSTER_ADMIN_PASSWORD))|((CLUSTER_ADMIN_AUTHORITIES))"]'
     properties.system_domain: '"((DOMAIN))"'
+    properties.ssl.skip_cert_verify: '((SKIP_CERT_VERIFY))'
     properties.traffic_controller.zone: '"((TRAFFIC_CONTROLLER_ZONE))"'
     properties.cf-usb.broker.external_url: '"brokers.((DOMAIN))"'
     properties.cf-usb.broker.username: '"((CF_USB_USERNAME))"'
@@ -1370,6 +1384,13 @@ configuration:
     properties.smoke_tests.apps_domain: '["((DOMAIN))"]'
     properties.smoke_tests.user: '"((CLUSTER_ADMIN_USERNAME))"'
     properties.smoke_tests.password: '"((CLUSTER_ADMIN_PASSWORD))"'
+    properties.hcf_set_proxy.api_proxy: ((HTTPS_PROXY))
+    properties.hcf_set_proxy.staging_http_proxy: ((HTTP_PROXY))
+    properties.hcf_set_proxy.staging_https_proxy: ((HTTPS_PROXY))
+    properties.hcf_set_proxy.staging_no_proxy: ((NO_PROXY))
+    properties.hcf_set_proxy.running_http_proxy: ((HTTP_PROXY))
+    properties.hcf_set_proxy.running_https_proxy: ((HTTPS_PROXY))
+    properties.hcf_set_proxy.running_no_proxy: ((NO_PROXY))
     # These settings must remain in the role manifest until they're available
     # in the upstream specs; even though they're also defined in the opinions file
     properties.cc.tweak_proc_sys: false

container-host-files/opt/hcf/bin/docker/configure_docker.sh

@@ -42,8 +42,14 @@ dopts="--storage-driver=devicemapper"
 dopts="$dopts --storage-opt dm.datadev=/dev/vg-docker/data"
 dopts="$dopts --storage-opt dm.metadatadev=/dev/vg-docker/metadata"
 
+for var in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY ; do
+  if test -n "${!var}" ; then
+    echo "export ${var}=${!var}" >> /etc/default/docker
+  fi
+done
+
 echo ___ Insert
-echo DOCKER_OPTS=\"$dopts\" | sudo tee -a /etc/default/docker
+echo DOCKER_OPTS=\"$dopts\" | tee -a /etc/default/docker
 
 # Activate the now-configured system