summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorThomas Wiest <twiest@users.noreply.github.com>2015-05-22 11:04:14 -0400
committerThomas Wiest <twiest@users.noreply.github.com>2015-05-22 11:04:14 -0400
commit5f784b42fb779d8feca2f4c4906966d159278811 (patch)
treebd22b603c9000656f9e885004c666e065e80cb49
parent09717b94a0b76d03db38baee793b141e1a651cdd (diff)
parent98acbb6d24921ded065fca3b57943ef1ae7fb15f (diff)
downloadopenshift-5f784b42fb779d8feca2f4c4906966d159278811.tar.gz
openshift-5f784b42fb779d8feca2f4c4906966d159278811.tar.bz2
openshift-5f784b42fb779d8feca2f4c4906966d159278811.tar.xz
openshift-5f784b42fb779d8feca2f4c4906966d159278811.zip
Merge pull request #233 from jsafrane/devel/nfs-volumes
Add nfs-volumes role.
-rw-r--r--roles/kube_nfs_volumes/README.md111
-rw-r--r--roles/kube_nfs_volumes/defaults/main.yml10
-rw-r--r--roles/kube_nfs_volumes/handlers/main.yml3
-rw-r--r--roles/kube_nfs_volumes/library/partitionpool.py240
-rw-r--r--roles/kube_nfs_volumes/meta/main.yml16
-rw-r--r--roles/kube_nfs_volumes/tasks/main.yml25
-rw-r--r--roles/kube_nfs_volumes/tasks/nfs.yml16
-rw-r--r--roles/kube_nfs_volumes/templates/nfs.json.j223
8 files changed, 444 insertions, 0 deletions
diff --git a/roles/kube_nfs_volumes/README.md b/roles/kube_nfs_volumes/README.md
new file mode 100644
index 000000000..965958bd6
--- /dev/null
+++ b/roles/kube_nfs_volumes/README.md
@@ -0,0 +1,111 @@
+# kube_nfs_volumes
+
+This role is useful to export disks as set of Kubernetes persistent volumes.
+It does so by partitioning the disks, creating ext4 filesystem on each
+partition, mounting the partitions, exporting the mounts via NFS and adding
+these NFS shares as NFS persistent volumes to existing Kubernetes installation.
+
+All partitions on given disks are used as the persistent volumes, including
+already existing partitions! There should be no other data (such as operating
+system) on the disks!
+
+## Requirements
+
+* Running Kubernetes with NFS persistent volume support (on a remote machine).
+
+* Works only on RHEL/Fedora-like distros.
+
+## Role Variables
+
+```
+# Options of NFS exports.
+nfs_export_options: "*(rw,no_root_squash,insecure,no_subtree_check)"
+
+# Directory, where the created partitions should be mounted. They will be
+# mounted as <mount_dir>/sda1 etc.
+mount_dir: /exports
+
+# Comma-separated list of disks to partition.
+# This role always assumes that all partitions on these disks are used as
+# physical volumes.
+disks: /dev/sdb,/dev/sdc
+
+# Whether to re-partition already partitioned disks.
+# Even though the disks won't get repartitioned on 'false', all existing
+# partitions on the disk are exported via NFS as physical volumes!
+foce: false
+
+# Specification of size of partitions to create. See library/partitionpool.py
+# for details.
+sizes: 100M
+
+# URL of Kubernetes API server, incl. port.
+kubernetes_url: https://10.245.1.2:6443
+
+# Token to use for authentication to the API server
+kubernetes_token: tJdce6Fn3cL1112YoIJ5m2exzAbzcPZX
+```
+
+## Dependencies
+
+None
+
+## Example Playbook
+
+With this playbook, `/dev/sdb` is partitioned into 100MiB partitions, all of
+them are mounted into `/exports/sdb<N>` directory and all these directories
+are exported via NFS and added as physical volumes to Kubernetes running at
+`https://10.245.1.2:6443`.
+
+ - hosts: servers
+ roles:
+ - role: kube_nfs_volumes
+ disks: "/dev/sdb"
+ sizes: 100M
+ kubernetes_url: https://10.245.1.2:6443
+ kubernetes_token: tJdce6Fn3cL1112YoIJ5m2exzAbzcPZX
+
+See library/partitionpool.py for details how `sizes` parameter can be used
+to create partitions of various sizes.
+
+## Full example
+Let's say there are two machines, 10.0.0.1 and 10.0.0.2, that we want to use as
+NFS servers for our Kubernetes cluster, running Kubernetes public API at
+https://10.245.1.2:6443.
+
+Both servers have three 1 TB disks, /dev/sda for the system and /dev/sdb and
+/dev/sdc to be partitioned. We want to split the data disks into 5, 10 and
+20 GiB partitions so that 10% of the total capacity is in 5 GiB partitions, 40%
+in 10 GiB and 50% in 20 GiB partitions.
+
+That means, each data disk will have 20x 5 GiB, 40x 10 GiB and 25x 20 GiB
+partitions.
+
+* Create an `inventory` file:
+ ```
+ [nfsservers]
+ 10.0.0.1
+ 10.0.0.2
+ ```
+
+* Create an ansible playbook, say `setupnfs.yaml`:
+ ```
+ - hosts: nfsservers
+ sudo: yes
+ roles:
+ - role: kube_nfs_volumes
+ disks: "/dev/sdb,/dev/sdc"
+ sizes: 5G:10,10G:40,20G:50
+ force: no
+ kubernetes_url: https://10.245.1.2:6443
+ kubernetes_token: tJdce6Fn3cL1112YoIJ5m2exzAbzcPZX
+ ```
+
+* Run the playbook:
+ ```
+ ansible-playbook -i inventory setupnfs.yml
+ ```
+
+## License
+
+Apache 2.0
diff --git a/roles/kube_nfs_volumes/defaults/main.yml b/roles/kube_nfs_volumes/defaults/main.yml
new file mode 100644
index 000000000..e296492f9
--- /dev/null
+++ b/roles/kube_nfs_volumes/defaults/main.yml
@@ -0,0 +1,10 @@
+---
+# Options of NFS exports.
+nfs_export_options: "*(rw,no_root_squash,insecure,no_subtree_check)"
+
+# Directory, where the created partitions should be mounted. They will be
+# mounted as <mount_dir>/sda1 etc.
+mount_dir: /exports
+
+# Force re-partitioning the disks
+force: false
diff --git a/roles/kube_nfs_volumes/handlers/main.yml b/roles/kube_nfs_volumes/handlers/main.yml
new file mode 100644
index 000000000..52f3ceffe
--- /dev/null
+++ b/roles/kube_nfs_volumes/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: restart nfs
+ service: name=nfs-server state=restarted
diff --git a/roles/kube_nfs_volumes/library/partitionpool.py b/roles/kube_nfs_volumes/library/partitionpool.py
new file mode 100644
index 000000000..1ac8eed4d
--- /dev/null
+++ b/roles/kube_nfs_volumes/library/partitionpool.py
@@ -0,0 +1,240 @@
+#!/usr/bin/python
+"""
+Ansible module for partitioning.
+"""
+
+# There is no pyparted on our Jenkins worker
+# pylint: disable=import-error
+import parted
+
+DOCUMENTATION = """
+---
+module: partitionpool
+short_description; Partition a disk into parititions.
+description:
+ - Creates partitions on given disk based on partition sizes and their weights.
+ Unless 'force' option is set to True, it ignores already partitioned disks.
+
+ When the disk is empty or 'force' is set to True, it always creates a new
+ GPT partition table on the disk. Then it creates number of partitions, based
+ on their weights.
+
+ This module should be used when a system admin wants to split existing disk(s)
+ into pools of partitions of specific sizes. It is not intended as generic disk
+ partitioning module!
+
+ Independent on 'force' parameter value and actual disk state, the task
+ always fills 'partition_pool' fact with all partitions on given disks,
+ together with their sizes (in bytes). E.g.:
+ partition_sizes = [
+ { name: sda1, Size: 1048576000 },
+ { name: sda2, Size: 1048576000 },
+ { name: sdb1, Size: 1048576000 },
+ ...
+ ]
+
+options:
+ disk:
+ description:
+ - Disk to partition.
+ size:
+ description:
+ - Sizes of partitions to create and their weights. Has form of:
+ <size1>[:<weigth1>][,<size2>[:<weight2>][,...]]
+ - Any <size> can end with 'm' or 'M' for megabyte, 'g/G' for gigabyte
+ and 't/T' for terabyte. Megabyte is used when no unit is specified.
+ - If <weight> is missing, 1.0 is used.
+ - From each specified partition <sizeX>, number of these partitions are
+ created so they occupy spaces represented by <weightX>, proportionally to
+ other weights.
+
+ - Example 1: size=100G says, that the whole disk is split in number of 100 GiB
+ partitions. On 1 TiB disk, 10 partitions will be created.
+
+ - Example 2: size=100G:1,10G:1 says that ratio of space occupied by 100 GiB
+ partitions and 10 GiB partitions is 1:1. Therefore, on 1 TiB disk, 500 GiB
+ will be split into five 100 GiB partition and 500 GiB will be split into fifty
+ 10GiB partitions.
+ - size=100G:1,10G:1 = 5x 100 GiB and 50x 10 GiB partitions (on 1 TiB disk).
+
+ - Example 3: size=200G:1,100G:2 says that the ratio of space occupied by 200 GiB
+ partitions and 100GiB partition is 1:2. Therefore, on 1 TiB disk, 1/3
+ (300 GiB) should be occupied by 200 GiB partitions. Only one fits there,
+ so only one is created (we always round nr. of partitions *down*). Teh rest
+ (800 GiB) is split into eight 100 GiB partitions, even though it's more
+ than 2/3 of total space - free space is always allocated as much as possible.
+ - size=200G:1,100G:2 = 1x 200 GiB and 8x 100 GiB partitions (on 1 TiB disk).
+
+ - Example: size=200G:1,100G:1,50G:1 says that the ratio of space occupied by
+ 200 GiB, 100 GiB and 50 GiB partitions is 1:1:1. Therefore 1/3 of 1 TiB disk
+ is dedicated to 200 GiB partitions. Only one fits there and only one is
+ created. The rest (800 GiB) is distributed according to remaining weights:
+ 100 GiB vs 50 GiB is 1:1, we create four 100 GiB partitions (400 GiB in total)
+ and eight 50 GiB partitions (again, 400 GiB).
+ - size=200G:1,100G:1,50G:1 = 1x 200 GiB, 4x 100 GiB and 8x 50 GiB partitions
+ (on 1 TiB disk).
+
+ force:
+ description:
+ - If True, it will always overwite partition table on the disk and create new one.
+ - If False (default), it won't change existing partition tables.
+
+"""
+
+# It's not class, it's more a simple struct with almost no functionality.
+# pylint: disable=too-few-public-methods
+class PartitionSpec(object):
+ """ Simple class to represent required partitions."""
+ def __init__(self, size, weight):
+ """ Initialize the partition specifications."""
+ # Size of the partitions
+ self.size = size
+ # Relative weight of this request
+ self.weight = weight
+ # Number of partitions to create, will be calculated later
+ self.count = -1
+
+ def set_count(self, count):
+ """ Set count of parititions of this specification. """
+ self.count = count
+
+def assign_space(total_size, specs):
+ """
+ Satisfy all the PartitionSpecs according to their weight.
+ In other words, calculate spec.count of all the specs.
+ """
+ total_weight = 0.0
+ for spec in specs:
+ total_weight += float(spec.weight)
+
+ for spec in specs:
+ num_blocks = int((float(spec.weight) / total_weight) * (total_size / float(spec.size)))
+ spec.set_count(num_blocks)
+ total_size -= num_blocks * spec.size
+ total_weight -= spec.weight
+
+def partition(diskname, specs, force=False, check_mode=False):
+ """
+ Create requested partitions.
+ Returns nr. of created partitions or 0 when the disk was already partitioned.
+ """
+ count = 0
+
+ dev = parted.getDevice(diskname)
+ try:
+ disk = parted.newDisk(dev)
+ except parted.DiskException:
+ # unrecognizable format, treat as empty disk
+ disk = None
+
+ if disk and len(disk.partitions) > 0 and not force:
+ print "skipping", diskname
+ return 0
+
+ # create new partition table, wiping all existing data
+ disk = parted.freshDisk(dev, 'gpt')
+ # calculate nr. of partitions of each size
+ assign_space(dev.getSize(), specs)
+ last_megabyte = 1
+ for spec in specs:
+ for _ in range(spec.count):
+ # create the partition
+ start = parted.sizeToSectors(last_megabyte, "MiB", dev.sectorSize)
+ length = parted.sizeToSectors(spec.size, "MiB", dev.sectorSize)
+ geo = parted.Geometry(device=dev, start=start, length=length)
+ filesystem = parted.FileSystem(type='ext4', geometry=geo)
+ part = parted.Partition(
+ disk=disk,
+ type=parted.PARTITION_NORMAL,
+ fs=filesystem,
+ geometry=geo)
+ disk.addPartition(partition=part, constraint=dev.optimalAlignedConstraint)
+ last_megabyte += spec.size
+ count += 1
+ try:
+ if not check_mode:
+ disk.commit()
+ except parted.IOException:
+ # partitions have been written, but we have been unable to inform the
+ # kernel of the change, probably because they are in use.
+ # Ignore it and hope for the best...
+ pass
+ return count
+
+def parse_spec(text):
+ """ Parse string with partition specification. """
+ tokens = text.split(",")
+ specs = []
+ for token in tokens:
+ if not ":" in token:
+ token += ":1"
+
+ (sizespec, weight) = token.split(':')
+ weight = float(weight) # throws exception with reasonable error string
+
+ units = {"m": 1, "g": 1 << 10, "t": 1 << 20, "p": 1 << 30}
+ unit = units.get(sizespec[-1].lower(), None)
+ if not unit:
+ # there is no unit specifier, it must be just the number
+ size = float(sizespec)
+ unit = 1
+ else:
+ size = float(sizespec[:-1])
+ spec = PartitionSpec(int(size * unit), weight)
+ specs.append(spec)
+ return specs
+
+def get_partitions(diskpath):
+ """ Return array of partition names for given disk """
+ dev = parted.getDevice(diskpath)
+ disk = parted.newDisk(dev)
+ partitions = []
+ for part in disk.partitions:
+ (_, _, pname) = part.path.rsplit("/")
+ partitions.append({"name": pname, "size": part.getLength() * dev.sectorSize})
+
+ return partitions
+
+
+def main():
+ """ Ansible module main method. """
+ module = AnsibleModule(
+ argument_spec=dict(
+ disks=dict(required=True, type='str'),
+ force=dict(required=False, default="no", type='bool'),
+ sizes=dict(required=True, type='str')
+ ),
+ supports_check_mode=True,
+ )
+
+ disks = module.params['disks']
+ force = module.params['force']
+ if force is None:
+ force = False
+ sizes = module.params['sizes']
+
+ try:
+ specs = parse_spec(sizes)
+ except ValueError, ex:
+ err = "Error parsing sizes=" + sizes + ": " + str(ex)
+ module.fail_json(msg=err)
+
+ partitions = []
+ changed_count = 0
+ for disk in disks.split(","):
+ try:
+ changed_count += partition(disk, specs, force, module.check_mode)
+ except Exception, ex:
+ err = "Error creating partitions on " + disk + ": " + str(ex)
+ raise
+ #module.fail_json(msg=err)
+ partitions += get_partitions(disk)
+
+ module.exit_json(changed=(changed_count > 0), ansible_facts={"partition_pool": partitions})
+
+# ignore pylint errors related to the module_utils import
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
+# import module snippets
+from ansible.module_utils.basic import *
+main()
+
diff --git a/roles/kube_nfs_volumes/meta/main.yml b/roles/kube_nfs_volumes/meta/main.yml
new file mode 100644
index 000000000..eb71a7a1f
--- /dev/null
+++ b/roles/kube_nfs_volumes/meta/main.yml
@@ -0,0 +1,16 @@
+---
+galaxy_info:
+ author: Jan Safranek
+ description: Partition disks and use them as Kubernetes NFS physical volumes.
+ company: Red Hat, Inc.
+ license: license (Apache)
+ min_ansible_version: 1.4
+ platforms:
+ - name: EL
+ versions:
+ - 7
+ - name: Fedora
+ versions:
+ - all
+ categories:
+ - cloud
diff --git a/roles/kube_nfs_volumes/tasks/main.yml b/roles/kube_nfs_volumes/tasks/main.yml
new file mode 100644
index 000000000..23b228d32
--- /dev/null
+++ b/roles/kube_nfs_volumes/tasks/main.yml
@@ -0,0 +1,25 @@
+---
+- name: Install pyparted (RedHat/Fedora)
+ yum: name=pyparted,python-httplib2 state=installed
+
+- name: partition the drives
+ partitionpool: disks={{ disks }} force={{ force }} sizes={{ sizes }}
+
+- name: create filesystem
+ filesystem: fstype=ext4 dev=/dev/{{ item.name }}
+ with_items: partition_pool
+
+- name: mount
+ mount: name={{mount_dir}}/{{ item.name }} src=/dev/{{ item.name }} state=mounted fstype=ext4 passno=2
+ with_items: partition_pool
+
+- include: nfs.yml
+
+- name: export physical volumes
+ uri: url={{ kubernetes_url }}/api/v1beta3/persistentvolumes
+ method=POST
+ body='{{ lookup("template", "../templates/nfs.json.j2") }}'
+ body_format=json
+ status_code=201
+ HEADER_Authorization="Bearer {{ kubernetes_token }}"
+ with_items: partition_pool
diff --git a/roles/kube_nfs_volumes/tasks/nfs.yml b/roles/kube_nfs_volumes/tasks/nfs.yml
new file mode 100644
index 000000000..87cf5f9a4
--- /dev/null
+++ b/roles/kube_nfs_volumes/tasks/nfs.yml
@@ -0,0 +1,16 @@
+---
+- name: Install NFS server on Fedora/Red Hat
+ yum: name=nfs-utils state=installed
+
+- name: Start rpcbind on Fedora/Red Hat
+ service: name=rpcbind state=started enabled=yes
+
+- name: Start nfs on Fedora/Red Hat
+ service: name=nfs-server state=started enabled=yes
+
+- name: Export the directories
+ lineinfile: dest=/etc/exports
+ regexp="^{{ mount_dir }}/{{ item.name }} "
+ line="{{ mount_dir }}/{{ item.name }} {{nfs_export_options}}"
+ with_items: partition_pool
+ notify: restart nfs
diff --git a/roles/kube_nfs_volumes/templates/nfs.json.j2 b/roles/kube_nfs_volumes/templates/nfs.json.j2
new file mode 100644
index 000000000..b42886ef1
--- /dev/null
+++ b/roles/kube_nfs_volumes/templates/nfs.json.j2
@@ -0,0 +1,23 @@
+{
+ "kind": "PersistentVolume",
+ "apiVersion": "v1beta3",
+ "metadata": {
+ "name": "pv-{{ inventory_hostname | regex_replace("\.", "-") }}-{{ item.name }}",
+ "labels": {
+ "type": "nfs"
+ }
+ },
+ "spec": {
+ "capacity": {
+ "storage": "{{ item.size }}"
+ },
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "NFS": {
+ "Server": "{{ inventory_hostname }}",
+ "Path": "{{ mount_dir }}/{{ item.name }}",
+ "ReadOnly": false
+ }
+ }
+}