Merge pull request #1 from openstf/coreos-sample

Add CoreOS Sample.

.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+user-data_*

README.md

@@ -1 +1,60 @@
-# setup-examples
+## Smartphone Test Farm Setup Examples
+
+[Smartphone Test Farm](https://github.com/openstf/stf) is a great tool to create on-premise device farm. And also, it is very easy to get started with it using it's `stf local` feature. But the problem is this feature was developed for debugging purpose. Users are not supposed to use it in production environment. It is okay to use it for a small farm of 10 ~ 15 devices with one host machine. But to scale the service you will have to deploy it on a computer cluster.
+
+This project will provide various setup examples for STF in production environment. I will be using [Vagrant](https://www.vagrantup.com/) with [VirtualBox](https://www.virtualbox.org/) provider to create virtual cluster for demonstration.
+
+## Assumptions
+- This project assumes that you already have read following nicely written documents. In case you haven't read, it is highly recommended to read them before going ahead.
+  - [STF README](https://github.com/openstf/stf/blob/master/README.md)
+  - [STF DEPLOYMENT DOC](https://github.com/openstf/stf/blob/master/doc/DEPLOYMENT.md)
+- You are on OS X
+  - *Theoritically this can be done on any OS which supports VirtualBox and Vagrant but they haven't been confirmed yet!*
+
+## Requirements
+- [VirtualBox](https://www.virtualbox.org/) >= 5.4.0
+- [Vagrant](https://www.vagrantup.com/) >= 1.7.3
+
+## Setup Architecture
+Before going any further, I am going to give you a brief idea about what kind of setups are we going to do. Most of the setups will have two major components.
+- Database Cluster
+  - Cluster for running database. [Rethinkdb](https://www.rethinkdb.com) cluster.
+- STF Cluster
+  - Cluster where STF components will be running. (CoreOS, Ubuntu etc)
+
+![stf_cluster_architecture](docs/stf_cluster_architecture.png)
+
+We will be creating these clusters using Vagrant and VirtualBox. And then deploy various software components on these clusters. By the end of this tutorial, you will have a running STF on your VMs which you will be able to access on http://172.17.8.101 from your browser. Stay Excited!
+
+## Setup
+
+```sh
+git clone https://github.com/openstf/setup-examples.git stf-setup-examples
+cd stf-setup-examples
+```
+
+## Create Rethinkdb Cluster
+
+Lets create rethinkdb cluster. Go to the `db` folder and run `vagrant up`. Yeah, thats it.
+
+```sh
+cd ./db; vagrant up
+```
+
+Above command will do following things
+- Download **ubuntu/trusty64** image if image is not present (*this may take time depending on internet speed*)
+- Launch Ubuntu VM and set its IP to `198.162.50.11`
+- Install and run rethinkdb server
+
+You can confirm if rethinkdb is up by visiting rethinkdb [admin console](http://198.162.50.11:8080)
+
+Please have a look at [Vagrantfile](db/Vagrantfile) to have better understanding about what is happening.
+
+**TODO**: Use 2 instances of VM for rethinkdb if somebody complains that this is not a real cluster :P
+
+## Create STF Cluster
+
+Please check following host specific docs for STF cluster creation and deployment strategy.
+
+- [CoreOS + Fleet](docs/coreos_fleet.md)
+- [Ubuntu](docs/ubuntu.md)

coreos/Vagrantfile

@@ -0,0 +1,90 @@
+# -*- mode: ruby -*-
+# -*- coding: utf-8 -*-
+
+require 'fileutils'
+require 'open-uri'
+require 'yaml'
+
+Vagrant.require_version '>= 1.6.0'
+
+CLOUD_CONFIG_PATH = File.join(File.dirname(__FILE__), 'user-data')
+
+update_channel = 'alpha'
+image_version = 'current'
+vm_memory = 512
+vm_cpus = 1
+
+cluster = [
+  { name: 'core-01', hostname: 'core-01', ip: '172.17.8.101', metadata: 'role=nginx'   },
+  { name: 'core-02', hostname: 'core-02', ip: '172.17.8.102', metadata: 'role=appside' },
+  { name: 'core-03', hostname: 'core-03', ip: '172.17.8.103', metadata: 'role=devside' }
+]
+
+# Used to fetch a new discovery token for a cluster of size 3
+new_discovery_url = "https://discovery.etcd.io/new?size=#{ cluster.size }"
+token = open(new_discovery_url).read
+
+Vagrant.configure('2') do |config|
+  # always use Vagrants insecure key
+  config.ssh.insert_key = false
+
+  config.vm.box = "coreos-%s" % update_channel
+  config.vm.box_url = "http://%s.release.core-os.net/amd64-usr/%s/coreos_production_vagrant.json" % [update_channel, image_version]
+
+  config.vm.provider :virtualbox do |v|
+    # On VirtualBox, we don't have guest additions or a functional vboxsf
+    # in CoreOS, so tell Vagrant that so it can be smarter.
+    v.check_guest_additions = false
+    v.functional_vboxsf     = false
+  end
+
+  cluster.each do |node|
+    config.vm.define node[:name] do |config|
+      config.vm.hostname = node[:hostname]
+
+      config.vm.provider :virtualbox do |v|
+        v.name = node[:name]
+        v.memory = vm_memory
+        v.cpus = vm_cpus
+
+        if node[:name] == 'core-01'
+          # Add usb filter to attach device
+          v.customize ['modifyvm', :id, '--usb', 'on']
+          # Run VBoxManage list usbhost command to get vendor_id and product_id of device
+          # Uncomment below line and add $VENDOR_ID and $PRODUCT_ID of your device
+          # In case of multiple devices, copy and paste below line.
+
+          # v.customize ['usbfilter', 'add', '0', '--target', :id, '--name', $ANY_NAME, '--vendorid', $VENDOR_ID, '--productid', $PRODUCT_ID]
+        end
+      end
+
+      config.vm.network :private_network, ip: node[:ip]
+
+      ## Provisioning
+      # cloudconfig-init
+      data = YAML.load(IO.readlines('user-data')[1..-1].join)
+
+      # Add discovery url
+      data['coreos']['etcd2']['discovery'] = token
+
+      # Add metadata
+      data['coreos']['fleet']['metadata'] = node[:metadata]
+
+      yaml = YAML.dump(data)
+
+      source_file = File.join(File.dirname(__FILE__), "user-data_#{ node[:name] }")
+      FileUtils.rm source_file if File.exists?(source_file)
+      File.open(source_file, 'w') { |file| file.write("#cloud-config\n\n#{ yaml }") }
+
+      config.vm.provision :file, source: source_file, destination: '/tmp/vagrantfile-user-data'
+      config.vm.provision :shell, inline: 'mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/', privileged: true
+
+      if node[:metadata] == 'role=nginx'
+        nginx_conf_file = File.join(File.dirname(__FILE__), 'nginx.conf')
+        config.vm.provision :file, source: nginx_conf_file, destination: '/tmp/nginx.conf'
+        config.vm.provision :shell, inline: 'mkdir -p /srv/nginx', privileged: true
+        config.vm.provision :shell, inline: 'mv /tmp/nginx.conf /srv/nginx/nginx.conf', privileged: true
+      end
+    end
+  end
+end

coreos/nginx.conf

@@ -0,0 +1,130 @@
+daemon off;
+worker_processes 4;
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  keepalive_timeout   65;
+  types_hash_max_size 2048;
+
+  default_type        application/octet-stream;
+
+  upstream stf_app {
+    # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+    server 172.17.8.10{1|2|3}:3100 max_fails=0;
+  }
+
+  upstream stf_auth {
+    # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+    server 172.17.8.10{1|2|3}:3200 max_fails=0;
+  }
+
+  upstream stf_storage_apk {
+    # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+    server 172.17.8.10{1|2|3}:3300 max_fails=0;
+  }
+
+  upstream stf_storage_image {
+    # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+    server 172.17.8.10{1|2|3}:3400 max_fails=0;
+  }
+
+  upstream stf_storage {
+    # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+    server 172.17.8.10{1|2|3}:3500 max_fails=0;
+  }
+
+  upstream stf_websocket {
+    # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+    server 172.17.8.10{1|2|3}:3600 max_fails=0;
+  }
+
+  types {
+    application/javascript  js;
+    image/gif               gif;
+    image/jpeg              jpg;
+    text/css                css;
+    text/html               html;
+  }
+
+  map $http_upgrade $connection_upgrade {
+    default  upgrade;
+    ''       close;
+  }
+
+  server {
+    listen       80 default_server;
+    listen       [::]:80 default_server;
+    server_name  stf.mydomain.org;
+    root         /dev/null;
+
+
+    resolver 8.8.4.4 8.8.8.8 valid=300s;
+    resolver_timeout 10s;
+
+    location ~ "^/d/1/([^/]+)/(?<port>[0-9]{5})/$" {
+      # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+      proxy_pass http://172.17.8.10{1|2|3}:$port/;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Real-IP $remote_addr;
+    }
+
+    location ~ "^/d/2/([^/]+)/(?<port>[0-9]{5})/$" {
+      # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+      proxy_pass http://172.17.8.10{1|2|3}:$port/;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Real-IP $remote_addr;
+    }
+
+    location ~ "^/d/3/([^/]+)/(?<port>[0-9]{5})/$" {
+      # PLEASE UPDATE IP ADDRESS WITH APPROPRIATE ONE
+      proxy_pass http://172.17.8.10{1|2|3}:$port/;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Real-IP $remote_addr;
+    }
+
+    location /s/image/ {
+      proxy_pass http://stf_storage_image;
+    }
+
+    location /s/apk/ {
+      proxy_pass http://stf_storage_apk;
+    }
+
+    location /s/ {
+      client_max_body_size 1024m;
+      client_body_buffer_size 128k;
+      proxy_pass http://stf_storage;
+    }
+
+    location /socket.io/ {
+      proxy_pass http://stf_websocket;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Real-IP $http_x_real_ip;
+    }
+
+    location /auth/ {
+      proxy_pass http://stf_auth/auth/;
+    }
+
+    location / {
+      proxy_pass http://stf_app;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Real-IP $http_x_real_ip;
+    }
+  }
+}

coreos/unit_files/adbd.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=ADB daemon
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=/usr/bin/docker pull sorccu/adb:latest
+ExecStartPre=-/usr/bin/docker kill %p
+ExecStartPre=-/usr/bin/docker rm %p
+ExecStart=/usr/bin/docker run --rm \
+  --name %p \
+  --privileged \
+  -v /dev/bus/usb:/dev/bus/usb \
+  --net host \
+  sorccu/adb:latest
+ExecStop=-/usr/bin/docker stop -t 2 %p
+
+[X-Fleet]
+Global=true

coreos/unit_files/nginx.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=STF nginx public load balancer
+After=docker.service
+Requires=docker.service
+ConditionPathExists=/srv/nginx/nginx.conf
+
+[Service]
+EnvironmentFile=/etc/environment
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=/usr/bin/docker pull nginx:1.7.10
+ExecStartPre=-/usr/bin/docker kill %p
+ExecStartPre=-/usr/bin/docker rm %p
+ExecStart=/usr/bin/docker run --rm \
+  --name %p \
+  --net host \
+  -v /srv/nginx/nginx.conf:/etc/nginx/nginx.conf:ro \
+  nginx:1.7.10 \
+  nginx
+ExecStop=/usr/bin/docker stop -t 2 %p
+
+[X-Fleet]
+MachineMetadata="role=nginx"

coreos/unit_files/rethinkdb-proxy-28015.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=RethinkDB proxy/28015
+After=docker.service
+Requires=docker.service
+
+[Service]
+EnvironmentFile=/etc/environment
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=/usr/bin/docker pull openstf/ambassador:latest
+ExecStartPre=-/usr/bin/docker kill %p
+ExecStartPre=-/usr/bin/docker rm %p
+ExecStart=/usr/bin/docker run --rm \
+  --name %p \
+  -p 28015 \
+  -e RETHINKDB_PORT_28015_TCP=tcp://198.162.50.11:28015 \
+  openstf/ambassador:latest
+ExecStop=-/usr/bin/docker stop -t 10 %p
+
+[X-Fleet]
+Global=true

coreos/unit_files/stf-app@.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=STF app
+After=rethinkdb-proxy-28015.service
+BindsTo=rethinkdb-proxy-28015.service
+
+[Service]
+EnvironmentFile=/etc/environment
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=/usr/bin/docker pull openstf/stf:latest
+ExecStartPre=-/usr/bin/docker kill %p-%i
+ExecStartPre=-/usr/bin/docker rm %p-%i
+ExecStart=/usr/bin/docker run --rm \
+  --name %p-%i \
+  --link rethinkdb-proxy-28015:rethinkdb \
+  -e "SECRET=answeri$42" \
+  -p %i:3000 \
+  openstf/stf:latest \
+  stf app --port 3000 \
+    --auth-url http://172.17.8.101/auth/mock/ \
+    --websocket-url http://172.17.8.101/
+ExecStop=-/usr/bin/docker stop -t 10 %p-%i

coreos/unit_files/stf-auth@.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=STF auth
+After=docker.service
+Requires=docker.service
+
+[Service]
+EnvironmentFile=/etc/environment
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=/usr/bin/docker pull openstf/stf:latest
+ExecStartPre=-/usr/bin/docker kill %p-%i
+ExecStartPre=-/usr/bin/docker rm %p-%i
+ExecStart=/usr/bin/docker run --rm \
+  --name %p-%i \
+  -e "SECRET=answeri$42" \
+  -p %i:3000 \
+  openstf/stf:latest \
+  stf auth-mock --port 3000 \
+    --app-url http://172.17.8.101/
+ExecStop=-/usr/bin/docker stop -t 10 %p-%i