Merge pull request #30 from codedellemc/ceph

Add Ceph + Rex Vagrant environment

ceph/.gitignore

@@ -0,0 +1,4 @@
+.vagrant
+*.vdi
+packer/packer_cache
+packer/*.box

ceph/README.md

@@ -0,0 +1,91 @@
+# REX-Ray + Ceph RBD
+
+This is a Vagrant environment using `VirtualBox` and
+`Virtual Media` as storage for Ceph to be consumed by REX-Ray along with
+Docker as the container runtime.  This can be used as a quick way
+to get started working with Ceph and REX-Ray.
+
+The `RBD` storage driver within REX-Ray is attaching RADOS Block Devices (RBD)
+to the Vagrant VMs.  This enables the persistent volumes to be
+moved between containers, which allow container hosts to be immutable and
+containers to remain non-persistent.
+
+## Installation
+
+### Clone the repo
+```
+git clone https://github.com/codedellemc/vagrant
+cd vagrant/ceph
+vagrant up
+```
+
+## Usage
+When the Vagrant environment is up and running, you can now run
+`vagrant ssh ceph-admin` to get into the VM.  Since REX-Ray requires root
+privileges for mounting, etc you can at this point issue something similar to
+`sudo -i` to ensure you are running as root.
+
+You can check the status of the Ceph cluster with `ceph -s`. You should be
+able to immediately run commands like `rexray volume create` and
+`rexray volume ls`, or do the same thing with docker via `docker volume create`
+and `docker volume ls`
+
+## Options
+There are optional fields in the `Vagrantfile` that can be modified or
+commented and uncommented.
+
+- Determine how many Ceph nodes to start
+ - `server_nodes`
+- Install the latest stable release
+ - `install_latest_stable_rex = true`
+- Install the latest staged release
+ - `install_latest_staaged_rex = true`
+- Install rex from from source
+ - `install_rex_from_source = true`
+
+Only one of the install options is intended to be set to true at one time.
+Setting multiple to true will result in wasted work, as each one overwrites the
+other. Setting all to false will result in no new rexray being installed,
+leaving whatever happens to be pre-installed in the vagrant box.
+
+When using `install_rex_from_source`, it is also possible to modify the
+`build_rexray` script to point to different branches or repo forks for both
+rexray and libstorage.
+
+## REX-Ray
+Consult the full REX-Ray documentation [here](http://rexray.readthedocs.org/en/stable/).
+
+Get information on the existing volumes:
+
+`rexray volume ls`
+
+Create a new volume:
+
+`rexray volume create --size=20 test`
+
+Delete a volume:
+
+`rexray volume remove test`
+
+## Docker
+Consult the Docker and REX-Ray documentation [here](http://rexray.readthedocs.io/en/stable/user-guide/schedulers/#docker).  
+
+You can also create volumes directly from the Docker CLI.  The
+following command created a volume of `20GB` size with name of
+`test`.
+
+```
+sudo docker volume create --driver=rexray --name=test --opt=size=20
+```
+
+Start a new container with a REX-Ray volume attached, and
+detach the volume when the container stops:
+
+```
+sudo docker run -it --volume-driver=rexray -v test:/test busybox /bin/sh
+# ls /
+bin   dev   etc   home  proc  root  sys   *test*  tmp   usr   var
+# exit
+```
+
+## That's it!

ceph/Vagrantfile

@@ -0,0 +1,123 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Number of Ceph server nodes
+server_nodes = 2
+
+# /24 network to use for private network
+network = "172.21.13"
+
+# Flags to control which REX-Ray to install
+# Setting all options to false will use whatever rexray is already present
+install_latest_stable_rex = false
+install_latest_staged_rex = false
+install_rex_from_source = true
+
+
+# Script to build rexray from source
+$build_rexray = <<SCRIPT
+export GOPATH=~/go
+SRC_DIR=${GOPATH}/src/github.com/codedellemc
+
+REX_REPO=https://github.com/codedellemc/rexray
+REX_BRANCH=master
+
+DEFAULT_LIBSTORAGE=false
+LS_REPO=https://github.com/codenrhoden/libstorage
+LS_BRANCH=feature/rbd
+
+sudo yum -y install golang
+
+mkdir -p ${SRC_DIR}
+cd ${SRC_DIR}
+
+if [ ${DEFAULT_LIBSTORAGE} = false ]; then
+	git clone ${LS_REPO} --branch ${LS_BRANCH}
+fi
+
+git clone ${REX_REPO} --branch ${REX_BRANCH}
+cd rexray
+
+if [ ${DEFAULT_LIBSTORAGE} = false ]; then
+	sysctl -w net.ipv4.ip_forward=1
+	DLOCAL_IMPORTS="github.com/codedellemc/libstorage" DGOOS=linux make && \
+	cp rexray /usr/bin
+else
+	make deps && make build && cp ${GOPATH}/bin/rexray /usr/bin
+fi
+SCRIPT
+
+
+
+Vagrant.configure("2") do |config|
+
+  # some shared setup
+  config.vm.box = "codedellemc/ceph_rexray"
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.ssh.forward_agent = true
+  config.ssh.insert_key = false
+  config.hostmanager.enabled = true
+
+  file_root = File.dirname(File.expand_path(__FILE__))
+
+  # We provision three nodes to be Ceph servers
+  (1..server_nodes).each do |i|
+    config.vm.define "ceph-server-#{i}" do |config|
+      config.vm.hostname = "ceph-server-#{i}"
+      config.vm.network :private_network, ip: "#{network}.#{i+10}"
+      config.vm.provider "virtualbox" do |vb|
+        vb.customize ["modifyvm", :id, "--macaddress1", "auto"]
+	vb.customize ["storagectl", :id, "--add", "sata", "--controller", "IntelAhci", "--name", "SATA", "--portcount", 30, "--hostiocache", "on"]
+        file_to_disk = File.join(file_root, "ceph#{i}osd.vdi")
+        unless File.exist?(file_to_disk)
+          vb.customize [
+            'createhd',
+            '--filename', file_to_disk,
+            '--format', 'VDI',
+            '--size', 10 * 1024
+          ]
+        end
+        vb.customize [
+          'storageattach', :id,
+          '--storagectl', 'SATA',
+          '--port', 1, '--device', 0,
+          '--type', 'hdd', '--medium',
+          file_to_disk
+        ]
+      end
+    end
+  end
+
+  # We need one Ceph admin machine to manage the cluster
+  config.vm.define "ceph-admin" do |admin|
+    admin.vm.hostname = "ceph-admin"
+    admin.vm.network :private_network, ip: "#{network}.10"
+    admin.vm.provider :virtualbox do |vb|
+      vb.customize ["modifyvm", :id, "--macaddress1", "auto"]
+      vb.customize ["storagectl", :id, "--add", "sata", "--controller", "IntelAhci", "--name", "SATA", "--portcount", 30, "--hostiocache", "on"]
+    end
+
+    if install_latest_stable_rex
+      admin.vm.provision "shell", privileged: true, inline: <<-SHELL
+        curl -sSL https://dl.bintray.com/emccode/rexray/install | sh -s stable
+      SHELL
+    end
+
+    if install_latest_staged_rex
+      admin.vm.provision "shell", privileged: true, inline: <<-SHELL
+        curl -sSL https://dl.bintray.com/emccode/rexray/install | sh -s staged
+      SHELL
+    end
+
+    if install_rex_from_source
+      admin.vm.provision "shell" do |s|
+        s.name = "build rexray"
+        s.privileged = true
+        s.inline = $build_rexray
+      end
+    end
+
+    admin.vm.provision "shell", privileged: false, path: "cephconfig.sh", args: server_nodes
+    admin.vm.provision "shell", privileged: true, path: "rexconfig.sh", args: server_nodes
+  end
+end

ceph/cephconfig.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -x
+
+NUM_NODES=$1
+
+if [ -e /etc/ceph/ceph.conf ]; then
+  echo "skipping Ceph config because it's been done before"
+  exit 0
+fi
+
+tee ~/.ssh/config << EOF
+Host *
+   StrictHostKeyChecking no
+   UserKnownHostsFile=/dev/null
+EOF
+
+chmod 0600 ~/.ssh/config
+
+#Configure Ceph
+
+if [ $NUM_NODES -ge 3 ]; then
+	ceph-deploy new ceph-server-1 ceph-server-2 ceph-server-3
+else
+	ceph-deploy new ceph-server-1
+fi
+
+if [ $NUM_NODES == 1 ]; then
+	tee -a ceph.conf << EOF
+osd pool default size = 1
+osd pool default min size = 1
+osd crush chooseleaf type = 0
+EOF
+
+elif [ $NUM_NODES == 2 ]; then
+	tee -a ceph.conf << EOF
+osd pool default size = 2
+osd pool default min size = 1
+EOF
+
+fi
+
+ceph-deploy mon create-initial
+for x in $(seq 1 $NUM_NODES); do
+	ceph-deploy osd create --zap-disk ceph-server-$x:/dev/sdb
+done
+ceph-deploy admin localhost

ceph/packer/http/ks.cfg

@@ -0,0 +1,44 @@
+install
+cdrom
+lang en_US.UTF-8
+keyboard us
+network --onboot yes --device eth0 --bootproto dhcp --noipv6
+rootpw --plaintext vagrant
+firewall --disabled
+authconfig --enableshadow --passalgo=sha512
+selinux --disabled
+timezone --utc Etc/UTC
+bootloader --location=mbr --driveorder=sda --append="crashkernel=auto rhgb quiet"
+
+text
+skipx
+zerombr
+
+clearpart --all --initlabel
+autopart
+
+firstboot --disabled
+reboot
+
+# Location of the package data
+url --url http://mirrors.usinternet.com/centos/7/os/x86_64/
+repo --name=updates --baseurl=http://mirrors.usinternet.com/centos/7/updates/x86_64/
+
+%packages --ignoremissing
+@core
+bzip2
+kernel-devel
+kernel-headers
+-ipw2100-firmware
+-ipw2200-firmware
+-ivtv-firmware
+%end
+
+%post
+/usr/bin/yum -y install sudo
+/usr/sbin/groupadd -g 501 vagrant
+/usr/sbin/useradd vagrant -u 501 -g vagrant -G wheel
+echo "vagrant"|passwd --stdin vagrant
+echo "vagrant        ALL=(ALL)       NOPASSWD: ALL" >> /etc/sudoers.d/vagrant
+chmod 0440 /etc/sudoers.d/vagrant
+%end

ceph/packer/scripts/base.sh

@@ -0,0 +1,4 @@
+sed -i "s/^.*requiretty/#Defaults requiretty/" /etc/sudoers
+yum install -y epel-release ntp git
+systemctl enable ntpd
+systemctl start ntpd

ceph/packer/scripts/ceph.sh

@@ -0,0 +1,5 @@
+yum install -y epel-release
+yum install -y python-pip
+pip install ceph-deploy
+ceph-deploy install --all --release jewel localhost
+chown vagrant:vagrant ~vagrant/ceph-deploy-ceph.log

ceph/packer/scripts/cleanup.sh

@@ -0,0 +1,9 @@
+yum -y erase gtk2 libX11 hicolor-icon-theme avahi freetype bitstream-vera-fonts
+yum -y clean all
+rm -rf VBoxGuestAdditions_*.iso
+rm -rf /tmp/rubygems-*
+
+dd if=/dev/zero of=/EMPTY bs=1M
+rm -f /EMPTY
+
+sync

ceph/packer/scripts/docker.sh

@@ -0,0 +1,3 @@
+curl -sSL https://get.docker.com/ | sh
+usermod -aG docker vagrant
+systemctl enable docker

ceph/packer/scripts/rexray.sh

@@ -0,0 +1 @@
+curl -sSL https://dl.bintray.com/emccode/rexray/install | sh -

ceph/packer/scripts/vagrant.sh

@@ -0,0 +1,6 @@
+date > /etc/vagrant_box_build_time
+
+mkdir -pm 700 /home/vagrant/.ssh
+curl -L https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub -o /home/vagrant/.ssh/authorized_keys
+chmod 0600 /home/vagrant/.ssh/authorized_keys
+chown -R vagrant:vagrant /home/vagrant/.ssh

ceph/packer/scripts/virtualbox.sh

@@ -0,0 +1,11 @@
+VBOX_VERSION=$(cat /home/vagrant/.vbox_version)
+
+# required for VirtualBox 4.3.26
+yum install -y bzip2
+
+cd /tmp
+mount -o loop /home/vagrant/VBoxGuestAdditions_$VBOX_VERSION.iso /mnt
+sh /mnt/VBoxLinuxAdditions.run
+umount /mnt
+rm -rf /home/vagrant/VBoxGuestAdditions_*.iso
+