From 1becded19ae6ca07e9f1483d8dcc14258bfb0b23 Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Tue, 27 Feb 2018 07:11:59 +0100 Subject: Kickstart --- kickstart/README | 9 ++ kickstart/autocd/build.sh | 17 +++ kickstart/autocd/isolinux/isolinux.cfg | 76 ++++++++++++++ kickstart/ipmi.sh | 144 ++++++++++++++++++++++++++ kickstart/kickstart/index.php | 183 +++++++++++++++++++++++++++++++++ kickstart/kickstart/ipekatrin-v2.ks | 82 +++++++++++++++ kickstart/kickstart/ipekatrin-v3.ks | 83 +++++++++++++++ kickstart/kickstart/ipekatrin.ks | 57 ++++++++++ kickstart/testvm/centos.sh | 9 ++ 9 files changed, 660 insertions(+) create mode 100644 kickstart/README create mode 100755 kickstart/autocd/build.sh create mode 100644 kickstart/autocd/isolinux/isolinux.cfg create mode 100755 kickstart/ipmi.sh create mode 100644 kickstart/kickstart/index.php create mode 100644 kickstart/kickstart/ipekatrin-v2.ks create mode 100644 kickstart/kickstart/ipekatrin-v3.ks create mode 100644 kickstart/kickstart/ipekatrin.ks create mode 100755 kickstart/testvm/centos.sh diff --git a/kickstart/README b/kickstart/README new file mode 100644 index 0000000..b686abc --- /dev/null +++ b/kickstart/README @@ -0,0 +1,9 @@ +Actions +======= +1) We need to build a CD which will request kickstart file from the web server +2) This CD should be programmed in IPMI interface + => Currently placed in /virtual/images/centos74-ands.iso on ipepdvsrv2 +3) The installation is triggered by ipmi commands +4) The web server runs a php script which detects connecting server and templates appropriate kickstart file + => It is installed on ufo.kit.edu in /srv/www/htdocs/ands/kickstart + => Detction is based on MAC address headers which are sent by CentOS CD diff --git a/kickstart/autocd/build.sh b/kickstart/autocd/build.sh new file mode 100755 index 0000000..289b4e2 --- /dev/null +++ b/kickstart/autocd/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ -n "bootcd/isolinux" ]; then + echo "1) Copy content of official CentOS boot CD to bootcd directory" + echo "2) Replace files provided in 'isolinux' directory" + exit +fi + +( + cd bootcd + chmod 664 isolinux/isolinux.bin + + rm -f ../centos74-ands.iso + mkisofs -o ../centos74-ands.iso -b isolinux/isolinux.bin -c boot.cat -no-emul-boot -V 'CentOS 7 x86_64' -boot-load-size 4 -boot-info-table -R -J -v -T . +) + +scp centos74-ands.iso root@192.168.26.134:/virtual/images/ diff --git a/kickstart/autocd/isolinux/isolinux.cfg b/kickstart/autocd/isolinux/isolinux.cfg new file mode 100644 index 0000000..e537309 --- /dev/null +++ b/kickstart/autocd/isolinux/isolinux.cfg @@ -0,0 +1,76 @@ +#ui vesamenu.c32 +#prompt 0 +#display boot.msg + +default auto +timeout 10 + +# Second port, 115200 baud +#serial 1 115200 + +label auto + menu label ^Kickstart CentOS 7 for Ands + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 console=tty1 console=ttyS1,115200 earlyprint=serial,ttyS1,115200 ip=dhcp inst.vnc inst.vncpassword=ipepdv inst.ks=http://ufo.kit.edu/ands/kickstart/ inst.ks.sendsn inst.ks.sendmac +# append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 nomodeset text console=tty1 console=ttyS1,115200 earlyprint=serial,ttyS1,115200 ip=dhcp inst.vnc inst.vncpassword=ipepdv inst.ks=http://ufo.kit.edu/ands/kickstart/ inst.ks.sendsn inst.ks.sendmac + +label linux + menu label ^Install CentOS 7 + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 quiet + +label check + menu label Test this ^media & install CentOS 7 + menu default + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 rd.live.check quiet + +menu separator # insert an empty line + +# utilities submenu +menu begin ^Troubleshooting + menu title Troubleshooting + +label vesa + menu indent count 5 + menu label Install CentOS 7 in ^basic graphics mode + text help + Try this option out if you're having trouble installing + CentOS 7. + endtext + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 xdriver=vesa nomodeset quiet + +label rescue + menu indent count 5 + menu label ^Rescue a CentOS system + text help + If the system will not boot, this lets you access files + and edit config files to try to get it booting again. + endtext + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 rescue quiet + +label memtest + menu label Run a ^memory test + text help + If your system is having issues, a problem with your + system's memory may be the cause. Use this utility to + see if the memory is working correctly. + endtext + kernel memtest + +menu separator # insert an empty line + +label local + menu label Boot from ^local drive + localboot 0xffff + +menu separator # insert an empty line +menu separator # insert an empty line + +label returntomain + menu label Return to ^main menu + menu exit + +menu end diff --git a/kickstart/ipmi.sh b/kickstart/ipmi.sh new file mode 100755 index 0000000..4ccd749 --- /dev/null +++ b/kickstart/ipmi.sh @@ -0,0 +1,144 @@ +user="ADMIN" +pass='$ipepdv$' +sleep=0.5 + +function smipmi_cmd { + echo "- Running: SMCIPMITool " + echo "$@" + /opt/smcipmi/SMCIPMITool "$@" +} + +function smipmi { + host=$1 + shift + smipmi_cmd $host ADMIN '$ipepdv$' "$@" +} + + +function ipmi_cmd { + echo -n "- Running: ipmitool " + echo "$@" + /usr/sbin/ipmitool "$@" +} + +function ipmi { + host=$1 + shift + ipmi_cmd -H $host -U ADMIN -P '$ipepdv$' "$@" + +} + +function configure { + host=$1 + + ipmi $host chassis bootdev disk persistent cons_redirect=enable verbose=default + sleep 0.5 +} + +function install { + host=$1 + +# Requires license +# smipmi $host wsiso mount 192.168.26.134 /images/centos74-ands.iso + + ipmi $host power off + sleep 10 + ipmi $host chassis bootdev cdrom + sleep $sleep + ipmi $host power on +} + +function boot { + host=$1 + + configure $host + ipmi $host power on + sleep $sleep +} + +function reboot { + host=$1 + + ipmi $host power off + sleep 10 + ipmi $host power on + sleep $sleep +} + + +function status { + host=$1 + + ipmi $host power status | grep "off" &> /dev/null + if [ $? -ne 0 ]; then echo 1; else echo 0; fi +} + +function wait_off { + host=$1 + + on=1 + while [ 1 ]; do + on=$(status $host) + [ "$on" -eq 0 ] && break + echo " - $host still running..." + sleep 5 + done +} + +function cmd { + ipmi "$@" +} + + +if [[ "$1" =~ ^[0-9\-]+$ ]]; then + IFS='-' read -ra range <<< "$1" + + if [ -n "${range[1]}" ]; then + servers=$(seq ${range[0]} ${range[1]}) + else + servers=$(seq ${range[0]} ${range[0]}) + fi + shift +else + servers=$(seq 1 3) +fi +iip=$(for i in $servers ; do echo "192.168.26.4$i" ; done) + +shift=1 +if [ -z "$1" ]; then + echo "$0 [#-#] " + echo "$0 [#] " + exit +elif [[ "$1" =~ config ]]; then + action="configure" +elif [[ "$1" =~ install ]]; then + action="install" +elif [[ "$1" =~ reboot ]]; then + action="reboot" +elif [[ "$1" =~ boot ]]; then + action="boot" +elif [[ "$1" =~ status ]]; then + action="status" +elif [[ "$1" =~ wait ]]; then + action="wait_off" +else + shift=0 + action="cmd" +fi + +if [ $shift -eq 1 ]; then + shift +fi + +for ip in $iip; do + eval "$action" "$ip" "$@" +done + +if [ $action = "install" ]; then + sleep 30 + for ip in $iip; do + wait_off "$ip" "$@" + configure "$ip" "$@" +# boot "$iip" "$@" + done +fi diff --git a/kickstart/kickstart/index.php b/kickstart/kickstart/index.php new file mode 100644 index 0000000..7ea2e54 --- /dev/null +++ b/kickstart/kickstart/index.php @@ -0,0 +1,183 @@ + "ipekatrin-v3.ks", + "domain" => "ipe.kit.edu", + "netmask" => "255.255.254.0", + "gw" => "141.52.64.207", + "ns" => "141.52.3.3,141.52.8.18", + "time" => "141.52.8.18", + "sysdisks" => "sdc,sdd", + "append_sol" => "console=tty1 console=ttyS1,115200 earlyprint=serial,ttyS1,115200", + "raid" => "RAID1", + "size" => "80000", + "bootsize" => "2048", + "ethdev" => "enp3s0f1", + "ethdev2" => "enp3s0f0", +); + +$SERVER_LIST = array( + "ipecsavm" => array_merge($KATRIN_SERVERS, array( + "ks" => "ipekatrin-v2.ks", + "macs" => array("66:66:66:13:13:00"), + "ip" => "192.168.26.254", + "netmask" => "255.255.255.0", + "gw" => "192.168.26.117", + "sysdisks" => "sda,sdb", + "raid" => "RAID0", + "size" => "60000", + "ethdev" => "link", + )), + "ipechilinga2" => array_merge($KATRIN_SERVERS, array( + "domain" => "ka.fzk.de", + "macs" => array("48:5b:39:75:fe:ec"), + "headers" => array( + "REMOTE_ADDR" => array("141.52.64.104") + ) + )), + "ipekatrin1" => array_merge($KATRIN_SERVERS, array( + "macs" => array("0c:c4:7a:de:f1:08", "0c:c4:7a:de:f1:09") + )), + "ipekatrin2" => array_merge($KATRIN_SERVERS, array( + "macs" => array("0c:c4:7a:de:f0:e6", "0c:c4:7a:de:f0:e7") + )), + "ipekatrin3" => array_merge($KATRIN_SERVERS, array( + "macs" => array("0c:c4:7a:a8:81:3e", "0c:c4:7a:a8:81:3f"), + "sysdisks" => "sdb,sdc", + "ethdev" => "eno2", + "ethdev2" => "eno1", + )) +); + + +function get_server($srvid) { + global $SERVER_LIST; + + $server = $SERVER_LIST[$srvid]; + + if (!isset($server["fqdn"])) + $server["fqdn"] = "{$srvid}.{$server['domain']}"; + + if (!isset($server["ip"])) + $server["ip"] = gethostbyname($server["fqdn"]); + + $disks = explode(",", $server["sysdisks"]); + if (!isset($server["bootdisk"])) + $server["bootdisk"] = $disks[0]; + + if (!isset($server["disk1"])) + $server["disk1"] = $disks[0]; + + if ((isset($disks[1]))&&(!isset($server["disk2"]))) + $server["disk2"] = $disks[1]; + + + unset($server["macs"]); + unset($server["headers"]); + + return $server; +} + +function find_mac($macs, $mac_header) { + if (!is_array($macs)) $macs = array($macs); + + foreach ($macs as $mac) { + if (preg_match("/$mac/", $mac_header)) + return true; + } + return false; +} + +function find_server_by_mac($mac_header) { + global $SERVER_LIST; + + foreach ($SERVER_LIST as $srvid => $server) { + if (find_mac($server['macs'], $mac_header)) + return get_server($srvid); + } + return false; +} + +function find_server_by_header($http_header, $value) { + global $SERVER_LIST; + + foreach ($SERVER_LIST as $srvid => $server) { + if ((is_array($server["headers"]))&&(isset($server["headers"][$http_header]))) { + $expected = $server["headers"][$http_header]; + if (!is_array($expected)) $expected = array($expected); + foreach ($expected as $re) { + if (preg_match("/$re/", $value)) + return get_server($srvid); + } + } + } + return false; +} + +function find_server() { + global $_SERVER; + + $headers = getallheaders(); + for ($i = 0; $i < 10; $i++) { + $if = "X-RHN-Provisioning-MAC-$i"; + if (!isset($headers[$if])) break; + + $server = find_server_by_mac($headers[$if]); + if ($server) return $server; + } + + foreach ($_SERVER as $header => $value) { + $server = find_server_by_header($header, $value); + if ($server) return $server; + } + + return false; +} + + + + +#echo "Request from: " . $_SERVER["REMOTE_ADDR"]; + +$server = find_server(); +if (!$server) { + $f = fopen("/srv/www/htdocs/ands/logs/kickstart-new.log", "a+"); + if ($f) { + fwrite($f, print_r($_SERVER, true)); + fwrite($f, print_r(getallheaders(), true)); + fclose($f); + } + return; +} + +$ks = file_get_contents($server["ks"]); + +$patterns=array(); $values=array(); +foreach ($server as $key => $val) { + array_push($patterns, "/@" . strtoupper($key) . "@/"); + array_push($values, $val); +} + +$ks = preg_replace($patterns, $values, $ks); + +if ($VERBOSE) { + $f = fopen("/srv/www/htdocs/ands/logs/kickstart.log", "a+"); + if ($f) { + fwrite($f, "-----------------------------------------------------\n"); + fwrite($f, print_r($server, true)); + fwrite($f, print_r($_SERVER, true)); + fwrite($f, print_r(getallheaders(), true)); + fwrite($f, "-----------------------------------------------------\n"); + fwrite($f, $ks); + fwrite($f, "=====================================================\n"); + fclose($f); + } +} + +header("Content-type: text/plain"); +echo $ks; + +?> diff --git a/kickstart/kickstart/ipekatrin-v2.ks b/kickstart/kickstart/ipekatrin-v2.ks new file mode 100644 index 0000000..5adc31f --- /dev/null +++ b/kickstart/kickstart/ipekatrin-v2.ks @@ -0,0 +1,82 @@ +#version=DEVEL + +# System authorization information +auth --enableshadow --passalgo=sha512 + +# Use CDROM installation media +cdrom + +# Use graphical install (graphical is enforce by vnc requested at kernel args) +#text +graphical + +# Run the Setup Agent on first boot +firstboot --enable +ignoredisk --only-use=@SYSDISKS@ +# Keyboard layouts +keyboard --vckeymap=us --xlayouts='us' +# System language +lang en_US.UTF-8 + +# Network information (device=link signifies first device link active) +network --device=link --bootproto=static --ip=@IP@ --netmask=@NETMASK@ --gateway=@GW@ --nameserver=@NS@ --noipv6 --onboot=on --activate +#network --bootproto=dhcp --device=eth0 --ipv6=auto --activate +network --hostname=@FQDN@ + + +# Partition clearing information +clearpart --all --drives=@SYSDISKS@ +zerombr + +# System bootloader configuration +bootloader --location=mbr --driveorder=@SYSDISKS@ --boot-drive=@BOOTDISK@ --append=" crashkernel=auto @APPEND_SOL@" + +#autopart --type=lvm +#reqpart --add-boot +part raid.01 --ondisk=@DISK1@ --asprimary --size @BOOTSIZE@ +part raid.02 --ondisk=@DISK2@ --asprimary --size @BOOTSIZE@ +part swap --ondisk=@DISK1@ --asprimary --fstype=swap --recommended +part swap --ondisk=@DISK2@ --asprimary --fstype=swap --recommended +part raid.03 --ondisk=@DISK1@ --asprimary --size @SIZE@ --grow +part raid.04 --ondisk=@DISK2@ --asprimary --size @SIZE@ --grow +raid /boot --level=@RAID@ --device md0 raid.01 raid.02 --fstype=ext4 +raid pv.01 --level=@RAID@ --device=md1 raid.03 raid.04 +volgroup sysvg pv.01 +logvol / --vgname=sysvg --size=@SIZE@ --name=lv_root --fstype=ext4 + +# Root password (new) +rootpw --iscrypted $6$ihAbktYN$T36KRAmi8ccjNrE5Y0gEl11Rb/dl3GjemejAJyHVzrAL51/st7aMZ0dqnMIkhubX/gUcPe5LdTlJODC9D/60h0 +# Root passowrd (old) +#rootpw --iscrypted $6$ioKrEQSxzYypx2HZ$jiynrl6knbmhbL066k.HjmxcwvQwBsT53LPlp2fRdkg2E1E7Gy4gwxaZ0m86rbD6q4dTaWdYfKhDVSij6N1Y7. + +# System services +services --enabled="chronyd" +# System timezone +timezone Europe/Berlin --isUtc --ntpservers=@TIME@ +user --groups=wheel --name=csa --gecos="Suren A. Chilingaryan" + +# SELinux configuration +#selinux --disabled + +# Do not configure the X Window System +skipx + +install +poweroff + + +%packages +@^minimal +@core +chrony +kexec-tools + +%end +%addon com_redhat_kdump --enable --reserve-mb='auto' +%end + +%anaconda +pwpolicy root --minlen=6 --minquality=1 --notstrict --nochanges --notempty +pwpolicy user --minlen=6 --minquality=1 --notstrict --nochanges --emptyok +pwpolicy luks --minlen=6 --minquality=1 --notstrict --nochanges --notempty +%end diff --git a/kickstart/kickstart/ipekatrin-v3.ks b/kickstart/kickstart/ipekatrin-v3.ks new file mode 100644 index 0000000..ae5aa35 --- /dev/null +++ b/kickstart/kickstart/ipekatrin-v3.ks @@ -0,0 +1,83 @@ +#version=DEVEL + +# System authorization information +auth --enableshadow --passalgo=sha512 + +# Use CDROM installation media +cdrom + +# Use graphical install (graphical is enforce by vnc requested at kernel args) +#text +graphical + +# Run the Setup Agent on first boot +firstboot --enable +ignoredisk --only-use=@SYSDISKS@ +# Keyboard layouts +keyboard --vckeymap=us --xlayouts='us' +# System language +lang en_US.UTF-8 + +# Network information (device=link signifies first device link active) +network --device=@ETHDEV@ --bootproto=static --ip=@IP@ --netmask=@NETMASK@ --gateway=@GW@ --nameserver=@NS@ --noipv6 --onboot=on --activate +#network --device=@ETHDEV2@ --bootproto=static --ip=@IP@ --netmask=@NETMASK@ --gateway=@GW@ --nameserver=@NS@ --noipv6 --onboot=off --activate +#network --bootproto=dhcp --device=eth0 --ipv6=auto --activate +network --hostname=@FQDN@ + + +# Partition clearing information +clearpart --all --drives=@SYSDISKS@ +zerombr + +# System bootloader configuration +bootloader --location=mbr --driveorder=@SYSDISKS@ --boot-drive=@BOOTDISK@ --append=" crashkernel=auto @APPEND_SOL@" + +#autopart --type=lvm +#reqpart --add-boot +part raid.01 --ondisk=@DISK1@ --asprimary --size @BOOTSIZE@ +part raid.02 --ondisk=@DISK2@ --asprimary --size @BOOTSIZE@ +part swap --ondisk=@DISK1@ --asprimary --fstype=swap --recommended +part swap --ondisk=@DISK2@ --asprimary --fstype=swap --recommended +part raid.03 --ondisk=@DISK1@ --asprimary --size @SIZE@ --grow +part raid.04 --ondisk=@DISK2@ --asprimary --size @SIZE@ --grow +raid /boot --level=@RAID@ --device md0 raid.01 raid.02 --fstype=ext4 +raid pv.01 --level=@RAID@ --device=md1 raid.03 raid.04 +volgroup sysvg pv.01 +logvol / --vgname=sysvg --size=@SIZE@ --name=lv_root --fstype=ext4 + +# Root password (new) +rootpw --iscrypted $6$ihAbktYN$T36KRAmi8ccjNrE5Y0gEl11Rb/dl3GjemejAJyHVzrAL51/st7aMZ0dqnMIkhubX/gUcPe5LdTlJODC9D/60h0 +# Root passowrd (old) +#rootpw --iscrypted $6$ioKrEQSxzYypx2HZ$jiynrl6knbmhbL066k.HjmxcwvQwBsT53LPlp2fRdkg2E1E7Gy4gwxaZ0m86rbD6q4dTaWdYfKhDVSij6N1Y7. + +# System services +services --enabled="chronyd" +# System timezone +timezone Europe/Berlin --isUtc --ntpservers=@TIME@ +user --groups=wheel --name=csa --gecos="Suren A. Chilingaryan" + +# SELinux configuration +#selinux --disabled + +# Do not configure the X Window System +skipx + +install +poweroff + + +%packages +@^minimal +@core +chrony +kexec-tools + +%end +%addon com_redhat_kdump --enable --reserve-mb='auto' +%end + +%anaconda +pwpolicy root --minlen=6 --minquality=1 --notstrict --nochanges --notempty +pwpolicy user --minlen=6 --minquality=1 --notstrict --nochanges --emptyok +pwpolicy luks --minlen=6 --minquality=1 --notstrict --nochanges --notempty +%end diff --git a/kickstart/kickstart/ipekatrin.ks b/kickstart/kickstart/ipekatrin.ks new file mode 100644 index 0000000..b394ca8 --- /dev/null +++ b/kickstart/kickstart/ipekatrin.ks @@ -0,0 +1,57 @@ +#version=DEVEL +# System authorization information +auth --enableshadow --passalgo=sha512 +# Use CDROM installation media +cdrom +# Use graphical install (graphical is enforce by vnc requested at kernel args) +#graphical +text + +# Run the Setup Agent on first boot +firstboot --enable +ignoredisk --only-use=@SYSDISKS@ +# Keyboard layouts +keyboard --vckeymap=us --xlayouts='us' +# System language +lang en_US.UTF-8 + +# Network information +#network --bootproto=dhcp --device=eth0 --ipv6=auto --activate +network --bootproto=static --ip=@IP@ --netmask=@NETMASK@ --gateway=@GW@ --nameserver=@NS@ --device=eth0 --ipv6=auto --activate +network --hostname=@FQDN@ + +# Root password +rootpw --iscrypted $6$ioKrEQSxzYypx2HZ$jiynrl6knbmhbL066k.HjmxcwvQwBsT53LPlp2fRdkg2E1E7Gy4gwxaZ0m86rbD6q4dTaWdYfKhDVSij6N1Y7. +# System services +services --enabled="chronyd" +# System timezone +timezone Europe/Berlin --isUtc --ntpservers=@TIME@ +user --groups=wheel --name=csa --password=$6$H8NeYbDfSuTtJKmA$pyUj57Ao1gAyT2C8ijivRxTMTkTClpOFUigmxsgKZ1L71Np6URTT4s6PU6WsuoEQgHgo9XjJNGePg/RneBY9a1 --iscrypted --gecos="Suren A. Chilingaryan" +# System bootloader configuration +bootloader --append=" crashkernel=auto" --location=mbr --boot-drive=@BOOTDISK@ +autopart --type=lvm +# Partition clearing information +clearpart --all --initlabel --drives=@SYSDISKS@ + + + +%packages +@^minimal +@core +chrony +kexec-tools + +%end + +%addon com_redhat_kdump --enable --reserve-mb='auto' + +%end + +%anaconda +pwpolicy root --minlen=6 --minquality=1 --notstrict --nochanges --notempty +pwpolicy user --minlen=6 --minquality=1 --notstrict --nochanges --emptyok +pwpolicy luks --minlen=6 --minquality=1 --notstrict --nochanges --notempty +%end + + +poweroff diff --git a/kickstart/testvm/centos.sh b/kickstart/testvm/centos.sh new file mode 100755 index 0000000..a437ed8 --- /dev/null +++ b/kickstart/testvm/centos.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +sudo modprobe kvm-intel +sudo /etc/init.d/vde start + +[ ! -f 1.qcow2 ] && qemu-img create -f qcow2 1.qcow2 "80G" +[ ! -f 2.qcow2 ] && qemu-img create -f qcow2 2.qcow2 "80G" + +qemu-system-x86_64 -enable-kvm -display sdl -hda 1.qcow2 -hdb 2.qcow2 -m 2048 -net nic,macaddr=66:66:66:13:13:00 -net vde,sock=/var/run/vde.ctl -cdrom centos74-ands.iso -boot order=d -- cgit v1.2.1