#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# -*- coding: utf-8 -*-
"""CloudStack Python utility library"""
import sys, os, subprocess, errno, re, time, glob
import urllib2
import xml.dom.minidom
import logging
import socket
# exit() error constants
E_GENERIC= 1
E_NOKVM = 2
E_NODEFROUTE = 3
E_DHCP = 4
E_NOPERSISTENTNET = 5
E_NETRECONFIGFAILED = 6
E_VIRTRECONFIGFAILED = 7
E_FWRECONFIGFAILED = 8
E_AGENTRECONFIGFAILED = 9
E_AGENTFAILEDTOSTART = 10
E_NOFQDN = 11
E_SELINUXENABLED = 12
try: E_USAGE = os.EX_USAGE
except AttributeError: E_USAGE = 64
E_NEEDSMANUALINTERVENTION = 13
E_INTERRUPTED = 14
E_SETUPFAILED = 15
E_UNHANDLEDEXCEPTION = 16
E_MISSINGDEP = 17
Unknown = 0
Fedora = 1
CentOS = 2
Ubuntu = 3
RHEL6 = 4
IPV4 = 4
IPV6 = 6
#=================== DISTRIBUTION DETECTION =================
if os.path.exists("/etc/fedora-release"): distro = Fedora
elif os.path.exists("/etc/centos-release"): distro = CentOS
elif os.path.exists("/etc/redhat-release"):
version = file("/etc/redhat-release").readline()
if version.find("Red Hat Enterprise Linux Server release 6") != -1:
distro = RHEL6
elif version.find("CentOS release") != -1:
distro = CentOS
else:
distro = CentOS
elif os.path.exists("/etc/legal") and "Ubuntu" in file("/etc/legal").read(-1): distro = Ubuntu
else: distro = Unknown
logFileName=None
# ================== LIBRARY UTILITY CODE=============
def setLogFile(logFile):
global logFileName
logFileName=logFile
def read_properties(propfile):
if not hasattr(propfile,"read"): propfile = file(propfile)
properties = propfile.read().splitlines()
properties = [ s.strip() for s in properties ]
properties = [ s for s in properties if
s and
not s.startswith("#") and
not s.startswith(";") ]
#[ logging.debug("Valid config file line: %s",s) for s in properties ]
proppairs = [ s.split("=",1) for s in properties ]
return dict(proppairs)
def stderr(msgfmt,*args):
"""Print a message to stderr, optionally interpolating the arguments into it"""
msgfmt += "\n"
if logFileName != None:
sys.stderr = open(logFileName, 'a+')
if args: sys.stderr.write(msgfmt%args)
else: sys.stderr.write(msgfmt)
def exit(errno=E_GENERIC,message=None,*args):
"""Exit with an error status code, printing a message to stderr if specified"""
if message: stderr(message,*args)
sys.exit(errno)
def resolve(host,port):
return [ (x[4][0],len(x[4])+2) for x in socket.getaddrinfo(host,port,socket.AF_UNSPEC,socket.SOCK_STREAM, 0, socket.AI_PASSIVE) ]
def resolves_to_ipv6(host,port):
return resolve(host,port)[0][1] == IPV6
###add this to Python 2.4, patching the subprocess module at runtime
if hasattr(subprocess,"check_call"):
from subprocess import CalledProcessError, check_call
else:
class CalledProcessError(Exception):
def __init__(self, returncode, cmd):
self.returncode = returncode ; self.cmd = cmd
def __str__(self): return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
subprocess.CalledProcessError = CalledProcessError
def check_call(*popenargs, **kwargs):
retcode = subprocess.call(*popenargs, **kwargs)
cmd = kwargs.get("args")
if cmd is None: cmd = popenargs[0]
if retcode: raise subprocess.CalledProcessError(retcode, cmd)
return retcode
subprocess.check_call = check_call
# python 2.4 does not have this
try:
any = any
all = all
except NameError:
def any(sequence):
for i in sequence:
if i: return True
return False
def all(sequence):
for i in sequence:
if not i: return False
return True
class Command:
"""This class simulates a shell command"""
def __init__(self,name,parent=None):
self.__name = name
self.__parent = parent
def __getattr__(self,name):
if name == "_print": name = "print"
return Command(name,self)
def __call__(self,*args,**kwargs):
cmd = self.__get_recursive_name() + list(args)
#print " ",cmd
kwargs = dict(kwargs)
if "stdout" not in kwargs: kwargs["stdout"] = subprocess.PIPE
if "stderr" not in kwargs: kwargs["stderr"] = subprocess.PIPE
popen = subprocess.Popen(cmd,**kwargs)
m = popen.communicate()
ret = popen.wait()
if ret:
e = CalledProcessError(ret,cmd)
e.stdout,e.stderr = m
raise e
class CommandOutput:
def __init__(self,stdout,stderr):
self.stdout = stdout
self.stderr = stderr
return CommandOutput(*m)
def __lt__(self,other):
cmd = self.__get_recursive_name()
#print " ",cmd," ",
zoneandpod = raw_input().strip()
if not zoneandpod:
# we go with default, do not touch anything, just break
return None
try:
# if parsing fails as an int, just vomit and retry
zoneandpod = int(zoneandpod)
if zoneandpod >= len(zonespods) or zoneandpod < 0: raise ValueError, "%s out of bounds"%zoneandpod
except ValueError,e:
stderr(str(e))
continue # re-ask
# oh yeah, the int represents an valid zone and pod index in the array
return zonespods[zoneandpod]
# this configures the agent
def device_exist(devName):
try:
alreadysetup = False
if distro in (Fedora,CentOS, RHEL6):
alreadysetup = augtool._print("/files/etc/sysconfig/network-scripts/ifcfg-%s"%devName).stdout.strip()
else:
alreadysetup = augtool.match("/files/etc/network/interfaces/iface",devName).stdout.strip()
return alreadysetup
except OSError,e:
return False
def setup_agent_config(configfile, host, zone, pod, cluster, guid, pubNic, prvNic):
stderr("Examining Agent configuration")
fn = configfile
text = file(fn).read(-1)
lines = [ s.strip() for s in text.splitlines() ]
confopts = dict([ m.split("=",1) for m in lines if "=" in m and not m.startswith("#") ])
confposes = dict([ (m.split("=",1)[0],n) for n,m in enumerate(lines) if "=" in m and not m.startswith("#") ])
if guid != None:
confopts['guid'] = guid
else:
if not "guid" in confopts:
stderr("Generating GUID for this Agent")
confopts['guid'] = uuidgen().stdout.strip()
if host == None:
try: host = confopts["host"]
except KeyError: host = "localhost"
stderr("Please enter the host name of the management server that this agent will connect to: (just hit ENTER to go with %s)",host)
print "> ",
newhost = raw_input().strip()
if newhost: host = newhost
confopts["host"] = host
if pubNic != None and device_exist(pubNic):
confopts["public.network.device"] = pubNic
if prvNic == None or not device_exist(prvNic):
confopts["private.network.device"] = pubNic
if prvNic != None and device_exist(prvNic):
confopts["private.network.device"] = prvNic
if pubNic == None or not device_exist(pubNic):
confopts["public.network.device"] = prvNic
stderr("Querying %s for zones and pods",host)
try:
if zone == None or pod == None:
x = list_zonespods(confopts['host'])
zoneandpod = prompt_for_hostpods(x)
if zoneandpod:
confopts["zone"],confopts["pod"] = zoneandpod
stderr("You selected zone %s pod %s",confopts["zone"],confopts["pod"])
else:
stderr("Skipped -- using the previous zone %s pod %s",confopts["zone"],confopts["pod"])
else:
confopts["zone"] = zone
confopts["pod"] = pod
confopts["cluster"] = cluster
except (urllib2.URLError,urllib2.HTTPError),e:
stderr("Query failed: %s. Defaulting to zone %s pod %s",str(e),confopts["zone"],confopts["pod"])
for opt,val in confopts.items():
line = "=".join([opt,val])
if opt not in confposes: lines.append(line)
else: lines[confposes[opt]] = line
text = "\n".join(lines)
file(fn,"w").write(text)
def setup_consoleproxy_config(configfile, host, zone, pod):
stderr("Examining Console Proxy configuration")
fn = configfile
text = file(fn).read(-1)
lines = [ s.strip() for s in text.splitlines() ]
confopts = dict([ m.split("=",1) for m in lines if "=" in m and not m.startswith("#") ])
confposes = dict([ (m.split("=",1)[0],n) for n,m in enumerate(lines) if "=" in m and not m.startswith("#") ])
if not "guid" in confopts:
stderr("Generating GUID for this Console Proxy")
confopts['guid'] = uuidgen().stdout.strip()
if host == None:
try: host = confopts["host"]
except KeyError: host = "localhost"
stderr("Please enter the host name of the management server that this console-proxy will connect to: (just hit ENTER to go with %s)",host)
print "> ",
newhost = raw_input().strip()
if newhost: host = newhost
confopts["host"] = host
stderr("Querying %s for zones and pods",host)
try:
if zone == None or pod == None:
x = list_zonespods(confopts['host'])
zoneandpod = prompt_for_hostpods(x)
if zoneandpod:
confopts["zone"],confopts["pod"] = zoneandpod
stderr("You selected zone %s pod %s",confopts["zone"],confopts["pod"])
else:
stderr("Skipped -- using the previous zone %s pod %s",confopts["zone"],confopts["pod"])
else:
confopts["zone"] = zone
confopts["pod"] = pod
except (urllib2.URLError,urllib2.HTTPError),e:
stderr("Query failed: %s. Defaulting to zone %s pod %s",str(e),confopts["zone"],confopts["pod"])
for opt,val in confopts.items():
line = "=".join([opt,val])
if opt not in confposes: lines.append(line)
else: lines[confposes[opt]] = line
text = "\n".join(lines)
file(fn,"w").write(text)
# =========================== DATABASE MIGRATION SUPPORT CODE ===================
# Migrator, Migratee and Evolvers -- this is the generic infrastructure.
class MigratorException(Exception): pass
class NoMigrationPath(MigratorException): pass
class NoMigrator(MigratorException): pass
INITIAL_LEVEL = '-'
class Migrator:
"""Migrator class.
The migrator gets a list of Python objects, and discovers MigrationSteps in it. It then sorts the steps into a chain, based on the attributes from_level and to_level in each one of the steps.
When the migrator's run(context) is called, the chain of steps is applied sequentially on the context supplied to run(), in the order of the chain of steps found at discovery time. See the documentation for the MigrationStep class for information on how that happens.
"""
def __init__(self,evolver_source):
self.discover_evolvers(evolver_source)
self.sort_evolvers()
def discover_evolvers(self,source):
self.evolvers = []
for val in source:
if hasattr(val,"from_level") and hasattr(val,"to_level") and val.to_level:
self.evolvers.append(val)
def sort_evolvers(self):
new = []
while self.evolvers:
if not new:
try: idx= [ i for i,s in enumerate(self.evolvers)
if s.from_level == INITIAL_LEVEL ][0] # initial evolver
except IndexError,e:
raise IndexError, "no initial evolver (from_level is None) could be found"
else:
try: idx= [ i for i,s in enumerate(self.evolvers)
if new[-1].to_level == s.from_level ][0]
except IndexError,e:
raise IndexError, "no evolver could be found to evolve from level %s"%new[-1].to_level
new.append(self.evolvers.pop(idx))
self.evolvers = new
def get_evolver_chain(self):
return [ (s.from_level, s.to_level, s) for s in self.evolvers ]
def get_evolver_by_starting_level(self,level):
try: return [ s for s in self.evolvers if s.from_level == level][0]
except IndexError: raise NoMigrator, "No evolver knows how to evolve the database from schema level %r"%level
def get_evolver_by_ending_level(self,level):
try: return [ s for s in self.evolvers if s.to_level == level][0]
except IndexError: raise NoMigrator, "No evolver knows how to evolve the database to schema level %r"%level
def run(self, context, dryrun = False, starting_level = None, ending_level = None):
"""Runs each one of the steps in sequence, passing the migration context to each. At the end of the process, context.commit() is called to save the changes, or context.rollback() is called if dryrun = True.
If starting_level is not specified, then the context.get_schema_level() is used to find out at what level the context is at. Then starting_level is set to that.
If ending_level is not specified, then the evolvers will run till the end of the chain."""
assert dryrun is False # NOT IMPLEMENTED, prolly gonna implement by asking the context itself to remember its state
starting_level = starting_level or context.get_schema_level() or self.evolvers[0].from_level
ending_level = ending_level or self.evolvers[-1].to_level
evolution_path = self.evolvers
idx = evolution_path.index(self.get_evolver_by_starting_level(starting_level))
evolution_path = evolution_path[idx:]
try: idx = evolution_path.index(self.get_evolver_by_ending_level(ending_level))
except ValueError:
raise NoEvolutionPath, "No evolution path from schema level %r to schema level %r" % \
(starting_level,ending_level)
evolution_path = evolution_path[:idx+1]
logging.info("Starting migration on %s"%context)
for ec in evolution_path:
assert ec.from_level == context.get_schema_level()
evolver = ec(context=context)
logging.info("%s (from level %s to level %s)",
evolver,
evolver.from_level,
evolver.to_level)
#try:
evolver.run()
#except:
#context.rollback()
#raise
context.set_schema_level(evolver.to_level)
#context.commit()
logging.info("%s is now at level %s",context,context.get_schema_level())
#if dryrun: # implement me with backup and restore
#logging.info("Rolling back changes on %s",context)
#context.rollback()
#else:
#logging.info("Committing changes on %s",context)
#context.commit()
logging.info("Migration finished")
class MigrationStep:
"""Base MigrationStep class, aka evolver.
You develop your own steps, and then pass a list of those steps to the
Migrator instance that will run them in order.
When the migrator runs, it will take the list of steps you gave him,
and, for each step:
a) instantiate it, passing the context you gave to the migrator
into the step's __init__().
b) run() the method in the migration step.
As you can see, the default MigrationStep constructor makes the passed
context available as self.context in the methods of your step.
Each step has two member vars that determine in which order they
are run, and if they need to run:
- from_level = the schema level that the database should be at,
before running the evolver
The value None has special meaning here, it
means the first evolver that should be run if the
database does not have a schema level yet.
- to_level = the schema level number that the database will be at
after the evolver has run
"""
# Implement these attributes in your steps
from_level = None
to_level = None
def __init__(self,context):
self.context = context
def run(self):
raise NotImplementedError
class MigrationContext:
def __init__(self): pass
def commit(self):raise NotImplementedError
def rollback(self):raise NotImplementedError
def get_schema_level(self):raise NotImplementedError
def set_schema_level(self,l):raise NotImplementedError