Monday, February 28, 2011

Importing old data into munin

This python script tries to figure out the original rrdtool create parameters that were used to create a given rrd. It's very basic (handles only RRA:Average afaik) so don't expect magic from it. It expects rrdtool to be your path.

Sample invocation:
lmwangi@jaghpu:~/rrd$ python rrdinfo-parser.py -f all.rrd
rrdtool create all.rrd --start 978300900 --step 300 \
DS:a:COUNTER:600:U:U \
DS:c:DERIVE:600:U:U \
DS:b:GAUGE:600:U:U \
DS:d:ABSOLUTE:600:U:U \
RRA:AVERAGE:0.5:1:10 \

lmwangi@jaghpu:~/rrd$ python rrdinfo-parser.py -f test.rrd
rrdtool create test.rrd --start 920804400 --step 300 \
DS:speed:COUNTER:600:U:U \
RRA:AVERAGE:0.5:1:24 \
RRA:AVERAGE:0.5:6:10 \

So say we have a Munin derived rrd that we need to import old data into. First we run the script to extract the schema, then we edit the --start parameter to an epoch timestamp that predates your data and finally recreate the rrd.
rrdtool create service-logs-total-g.rrd --start 1108598400 --step 300 \                                                   
DS:42:GAUGE:600:U:U \
RRA:AVERAGE:0.5:1:576 \
RRA:MIN:0.5:1:576 \
RRA:MAX:0.5:1:576 \
RRA:AVERAGE:0.5:6:432 \
RRA:MIN:0.5:6:432 \
RRA:MAX:0.5:6:432 \
RRA:AVERAGE:0.5:24:540 \
RRA:MIN:0.5:24:540 \
RRA:MAX:0.5:24:540 \
RRA:AVERAGE:0.5:288:450
Then we import the processed data (in my case, log file summaries) which looks like:
head 2010.data
1262296800:4241:221:173:276
1262297400:3920:197:155:231
1262298000:4171:184:226:208
1262298600:3700:197:159:244
1262299200:3350:195:166:227
where the fields are ts:total:errors:unknowns:etc.
Using a simple awk script we extract field one and two (timestamp and total) and import them into the rrd and when we are done, overwrite the original rrd.
FILE=service-logs-total-g.rrd
for record in $(cat 2010.data|awk -F: '{print $1":"$2}');
do
rrdupdate $FILE $record;
done

Poweroff or poweron usb devices in Linux

Sometimes, I need my external drive to suspend/spin down when I am suspending my machine.
Use lsusb to replace the vendor/product ids.
In my case
$ lsusb
...
Bus 001 Device 008: ID 1058:0704 Western Digital Technologies, Inc. Passport External HDD
...



~/bin/poweroff-usbdisk.sh

#!/bin/sh
# Poweroff a device identified by the vendor/product id below.
# If you have two similar devices (e.g two western digital drives),
# the script will fail. The fix is easy: enumerate the devices
# and suspend them individually. 

vendorid=1058
productid=0704

# Look for western digital
USBDIR=$(dirname $(find  /sys/bus/usb/devices/usb[1-5]/ -iname "*vendor*"|xargs grep $vendorid|cut -f1 -d: ))

# Verify it's the hdd
grep $productid $USBDIR/idProduct

#Power off 
if [ $? -eq 0 ]; then
sudo umount /mnt/audio/ && echo suspend |sudo tee $USBDIR/power/level;
fi
~/bin/poweron-usbdisk.sh
#!/bin/sh
# Poweron a device identified by the vendor/product id below.

# Look for western digital
USBDIR=$(dirname $(find  /sys/bus/usb/devices/usb[1-5]/ -iname "*vendor*"|xargs grep $vendorid |cut -f1 -d: ))
vendorid=1058
productid=0704


# Verify it's the hdd
grep $productid $USBDIR/idProduct

#Power off 
if [ $? -eq 0 ]; then  echo on |sudo tee $USBDIR/power/level; fi

Wednesday, February 16, 2011

A simple munin graphing script that uses mk-heartbeat to monitor replication lag between two MySQL servers. Place it in your /etc/munin/plugins/ directory and configure your environment.
/etc/munin/plugin-conf.d//mysql
[mysql_*]
env.mysqlconnection DBI:mysql:mysql
env.mysqluser Reporter
env.mysqlpassword Blah


/etc/munin/plugins/mysql_replication_lag

#!/bin/sh
#
# Plugin to monitor the mysql replication lag
#
# Parameters:
#
# config (required)
# autoconf (optional - used by munin-config)
#
# $Log$
# Revision 1: Tue Feb 15 17:04:59 MUT 2011
# Initial writeup and testing
#
#
#
# Magic markers (Used by munin-config and some installation scripts.
# Optional):
#
#%# family=auto
#%# capabilities=autoconf

if [ "$1" = "autoconf" ]; then
echo yes
exit 0
fi

if [ "$1" = "config" ]; then

echo 'graph_title Replication Lag'
echo 'graph_args --base 1000 -l 0'
echo 'graph_vlabel Replication lag in seconds'
echo 'graph_category mysql'
echo 'graph_info This graph monitors the replication lag using mk-heartbeat.'
echo 'lag.label Lag in seconds'
echo 'lag.warning :300'
echo 'lag.warning :3600'
echo 'lag.info The number of currently open files.'
exit 0
fi
echo -n "lag.value "
mysql -u ${mysqluser} -p${mysqlpassword} -B -N -e "select now()-ts as replication_lag from maatkit.heartbeat;"

Thursday, February 10, 2011

Cloning a VM that uses LVM using libvirt utilities

Get specs for your existing VM storage and create an LV for the new VM

# lvdisplay /dev/sdbvg/eng3.example.net_root
--- Logical volume ---
LV Name /dev/sdbvg/eng3.example.net_root
VG Name sdbvg
LV UUID 9APovE-4ad5-QYe9-UB3U-ndP2-lW3p-IcZWt1
LV Write Access read/write
LV Status available
# open 1
LV Size 40.00 GB
Current LE 10240
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:46

[root@vm01 ~]# lvcreate -L 40G -n template.example.net_swap /dev/sdbvg
Logical volume "template.example.net_swap" created

Do the same thing for swap or any other partitions you may have.
Then call virt-clone
virt-clone --prompt --original eng3.example.net --name template.example.net --file /dev/sdbvg/template.example.net_root --file/dev/sdbvg/template.example.net_swap
Cloning /dev/sdbvg/eng3..example.net_root | 40 GB 05:55
Cloning /dev/sdbvg/eng3.example.nett_swap | 4.0 GB 00:34



Updating the Soekris firmware

Prepare the Soekris

Get/Build a null modem cable

SignalDE-9DE-9Signal
FG (Frame Ground)--FG
TD (Transmit Data)23RD
RD (Receive Data)32TD
RTS (Request To Send)78CTS
CTS (Clear To Send)87RTS
SG (Signal Ground)55SG
DSR (Data Set Ready)64DTR
DTR (Data Terminal Ready)46DSR

Setting up minicom for Soekris serial

+-----------------------------------------------------------------------+
| A - Serial Device : /dev/ttyS0 |
| B - Lockfile Location : /var/lock |
| C - Callin Program : |
| D - Callout Program : |
| E - Bps/Par/Bits : 57600 8N1 |
| F - Hardware Flow Control : No |
| G - Software Flow Control : No |
| |
| Change which setting? |
+-----------------------------------------------------------------------+

Sample flash update and reboot process - bios
comBIOS ver. 1.33c 20080626 Copyright (C) 2000-2008 Soekris Engineering.

net5501

0512 Mbyte Memory CPU Geode LX 500 Mhz

Pri Mas SanDisk SDCFH2-002G LBA Xlt 992-64-63 2001 Mbyte

Slot Vend Dev ClassRev Cmd Stat CL LT HT Base1 Base2 Int
-------------------------------------------------------------------
0:01:2 1022 2082 10100000 0006 0220 08 00 00 A0000000 00000000 10
0:06:0 1106 3053 02000096 0117 0210 08 40 00 0000E101 A0004000 11
0:07:0 1106 3053 02000096 0117 0210 08 40 00 0000E201 A0004100 05
0:08:0 1106 3053 02000096 0117 0210 08 40 00 0000E301 A0004200 09
0:09:0 1106 3053 02000096 0117 0210 08 40 00 0000E401 A0004300 12
0:20:0 1022 2090 06010003 0009 02A0 08 40 80 00006001 00006101
0:20:2 1022 209A 01018001 0005 02A0 08 00 00 00000000 00000000
0:21:0 1022 2094 0C031002 0006 0230 08 00 80 A0005000 00000000 15
0:21:1 1022 2095 0C032002 0006 0230 08 00 00 A0006000 00000000 15

4 Seconds to automatic boot. Press Ctrl-P for entering Monitor.

comBIOS Monitor. Press ? for help.

> ?
comBIOS Monitor Commands

boot [drive][:partition] INT19 Boot
reboot cold boot
download download a file using XMODEM/CRC
flashupdate update flash BIOS with downloaded file
time [HH:MM:SS] show or set time
date [YYYY/MM/DD] show or set date
d[b|w|d] [adr] dump memory bytes/words/dwords
e[b|w|d] adr value [...] enter bytes/words/dwords
i[b|w|d] port input from 8/16/32-bit port
o[b|w|d] port value output to 8/16/32-bit port
run adr execute code at adr
cmosread [adr] read CMOS RAM data
cmoswrite adr byte [...] write CMOS RAM data
cmoschecksum update CMOS RAM Checksum
set parameter=value set system parameter to value
show [parameter] show one or all system parameters
?/help show this help
>
> download -
Start sending file using XMODEM/CRC protocol.
File downloaded succesfully, size 784 Blocks.
> flashupdate
Updating BIOS Flash ,,,,,,,,,,,,,,,,,,,,,,,,,,,,..,,,,.... Done.
> reboot
POST: 012345689bcefghips1234ajklnopqr,,,tvwxy

Wednesday, February 9, 2011

Libvirt utilities over TLS

If you are running libvirt on a different host and you don't trust the devices in your network path, you should use TLS. You can of course use ssh :). To use TLS/SSL with libvirt, you should have:
  • Access to CA henceforth known as 'the CA'.
  • Server key, server cert signed by the CA.
  • Client key, client cert signed by the CA.
If you don't have a CA, it's easy to generate one using openSSL or better use certtool (man 1 certtool) from gnutls binaries.

Once you have a CA, have a look at the libvirtd help on your server.
xen02:~/pki# libvirtd -h
libvirtd: invalid option -- 'h'
Usage:
libvirtd [options]
Options:
-v | --verbose Verbose messages.
-d | --daemon Run as a daemon & write PID file.
-l | --listen Listen for TCP/IP connections.
-t | --timeout Exit after timeout period.
-f | --config Configuration file.
| --version Display version information.
-p | --pid-file Change name of PID file.
.......
TLS:
CA certificate: /etc/pki/CA/cacert.pem
Server certificate: /etc/pki/libvirt/servercert.pem
Server private key: /etc/pki/libvirt/private/serverkey.pem
.......
From the output above, it's evident that you will need to copy the CA's certificate to /etc/pki/CA/cacert.pem, generate a server key, a server CSR then get the CSR to the CA. The CA will then provision you with a server cert which you can place in the appropriate path.

You will need to enable TLS in /etc/libvirt/libvirtd.conf
listen_tls = 1
You will probably want to enable TLS authentication for VNC as well as listening on interfaces other than localhost:
vnc_listen = "0.0.0.0"
vnc_tls = 1
#Read the documentation for this
vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc"
vnc_tls_x509_verify = 1
Here's my PKI directory
PKI directory listing
$ find /etc/pki/
/etc/pki/
/etc/pki/libvirt-vnc
/etc/pki/libvirt-vnc/ca-cert.pem
/etc/pki/libvirt-vnc/server-key.pem
/etc/pki/libvirt-vnc/server-cert.pem
/etc/pki/libvirt
/etc/pki/libvirt/servercert.pem
/etc/pki/libvirt/private
/etc/pki/libvirt/private/serverkey.pem
/etc/pki/CA
/etc/pki/CA/cacert.pem
To make life easier, You can make VNC and libvirt to share the same keys and certs.
Restart the libvirt service
/etc/init.d/libvirt-bin restart
The client requires a similar PKI setup. The same CA should sign the server and client certs.
$ find /etc/pki/
/etc/pki/
/etc/pki/CA
/etc/pki/CA/cacert.pem
/etc/pki/libvirt
/etc/pki/libvirt/private
/etc/pki/libvirt/private/clientkey.pem
/etc/pki/libvirt/clientcert.pem
Add a connection from your virt-manager that utilizes TLS or use the virsh tools:
virt-viewer -c qemu+tls://xen02.example.net/system box2
virsh -c qemu+tls://xen02.example.net/system list
Id Name State
----------------------------------
7 box1 running
10 box2 running
11 box3 running
13 box4 running
14 box5 running
15 box6 running
16 box7 running


Saturday, February 5, 2011

Google Translate using Python


A simple google translator for your console: It uses the labs api described here
http://code.google.com/apis/language/translate/v2/using_rest.html. The latest code is here.

#!/usr/bin/env python
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 textwidth=79 autoindent

"""
Python source code

Simple translator using the google translate API.
Example run:
    $ python translate1.py  -t sw "Hello. Are we meeting today?"
    Hello. Sisi ni mkutano wa leo?
"""
import urllib2
import json
import optparse

# API KEY
API_KEY = "Replace with your Key"
API_URL = "https://www.googleapis.com/language/translate/v2?\
key=%s&q=%s&source=%s&target=%s&prettyprint=false"

#Defaults
DEFAULT_SRC_LANG = "en"
DEFAULT_DEST_LANG = "fr"


class Translate(object):

"""Translate: Uses the google api to translate a string from one language
    to another
    """
def __init__(self):
super(Translate, self).__init__()
self.langs = ["af", "sq", "ar", "be", "bg", "ca", "zh-CN", "zh-TW",
"hr", "cs", "da", "nl", "en", "et", "tl", "fi", "fr",
"gl", "de", "el", "ht", "iw", "hi", "hu", "is", "id",
"ga", "it", "ja", "lv", "lt", "mk", "ms", "mt", "no",
"fa", "pl", "pt", "ro", "ru", "sr", "sk", "sl", "es",
"sw", "sv", "th", "tr", "uk", "vi", "cy", "yi"]

self.uri = API_URL

def translate(self, params):
"""Translates texts
        keywords:
            params - Dictionary
                src_text - String
                src_lang - 2 letter iso code for language
                dest_lang - 2 letter iso code for language
        """
req_uri = self.uri % (API_KEY, urllib2.quote(params['src_text']),
params['src_lang'],
params['dest_lang'])

hdl = urllib2.urlopen(req_uri)
resp = hdl.read()
hdl.close()
j = json.loads(resp)
try:
return j['data']['translations'][0]['translatedText']
except TypeError:
return ""


def check_lang(option, opt_str, value, parser):
""" Callback for optparse. Verifies that value is an item in a list"""
translator = Translate()
langs = translator.langs
if value not in langs:
raise optparse.OptionValueError(
"Invalid option: %s.\nLanguage not in %s" % (opt_str, langs))

setattr(parser.values, option.dest, value)


def main():
"""Main function. Called when this file is a shell script"""
translator = Translate()
usage = "usage: %prog [options] 'Text to translate'"
parser = optparse.OptionParser(usage)
parser.add_option("-f", "--from", action="callback",
callback=check_lang, dest="src_lang",
default=DEFAULT_SRC_LANG, type="string",
help="Translate from this language. Default is %default")

parser.add_option("-t", "--to", action="callback",
default=DEFAULT_DEST_LANG, type="string",
callback=check_lang, dest="dest_lang",
help="Translate to this language. Default is %default")

(options, args) = parser.parse_args()

params = {}
params['src_lang'] = options.src_lang
params['dest_lang'] = options.dest_lang
params['src_text'] = args[0]
dest_text = translator.translate(params)
print dest_text

if __name__ == '__main__':
main()