利用Fabric+Capistrano实现Python自动化部署

Fabric是一个用于应用（批量）部署和系统（批量）管理的Python库和命令行工具，关于Fabric的介绍请参考：http://www.fabfile.org/。
Capistrano是一个用Ruby语言编写的远程服务器自动化和部署工具，关于Capistrano的介绍请参考：http://capistranorb.com/。
本文仅使用Python语言和部分Linux或Windows系统命令，借助Fabric模块和Capistrano的部署思路，实现在Linux平台和Windows平台的自动化部批量署应用或实现批量系统管理（批量执行命令，批量上传文件等），其中Fabric部分利用Fabric的模块，Capistrano部分用Python语言按照Capistrano的部署思路“重写（Python实现Capistrano）”。
关于Capistrano的“重写”说明。Capistrano使用Ruby语言写的，在部署很多应用上有很大的优势，个人认为它设计最好的部分就是它的目录结构。目录结构的详细信息可以参考：http://capistranorb.com/documentation/getting-started/structure/#。有了这个目录结构可以轻松实现每一个部署版本的备份与回滚，之前用Bash Shell“重写”过一次，可以参考本文《Linux Shell脚本之远程自动化部署java maven项目 》，这次相当于用Python重写一下（Capistrano还有大量精髓的东西，本文算是抛砖引玉，其他的日后再发掘），毕竟Shell脚本不容易实现像Fabric那样的批量操作。
本文的demo是将https://github.com/DingGuodong/GoogleHostsFileForLinux.git 中的用于翻墙访问Google的脚本上传到指定的服务器，以在Windows操作为例，先在本地生成Capistrano目录结构，再把git项目clone到本地，将脚本文件从repo目录下抽出，放到current目录下，current是release目录下某个时间戳的软链接（Windows下测试可能有些奇怪，因为Windows下没法做软连接，用Python创建快捷方式暂时没有找到方法），再将此脚本通过Fabric的put命令上传到指定的远程服务器上。
demo在脚本中的位置可以从TODO中找到，由于fabric需要通过fab命令+脚本执行，因此在脚本的最后使用了terminal_debug()函数实现脚本执行，如果对python和fabric很熟悉，那么将下文的脚本放到pycharm中打开，略微看看脚本，即时没有注释（有人曾说，好的代码是不写注释的，虽然代码写的不好，但至少要朝着这个目标努力）也能看的很明白。其实在真正了解了Fabric和Capistrano后，重新阅读这个脚本或者看这篇文章一定觉得确实写的很普（渣）通（渣）。
脚本的部分说明（时间原因就不展开写了，可以等熟悉了Fabric和Capistrano后再看）：
此脚本文件的开始几行配置有名字为config的字典，主要用于声明deploy_to、repo_url和branch以及keep_releases；

env变量用于向Fabric声明主机信息。

运行结果：



与Capistrano相同的目录结构：



模仿Capistrano生成的相同格式的日志：



脚本内容可以从GitHub上获取：https://github.com/DingGuodong/LinuxBashShellScriptForOps/blob/master/projects/autoOps/pythonSelf/pyCapistrano.py
脚本内容如下：

#!/usr/bin/python
# encoding: utf-8
# -*- coding: utf8 -*-
"""
Created by PyCharm.
File:               LinuxBashShellScriptForOps:TestGit.py
User:               Guodong
Create Date:        2016/8/24
Create Time:        9:40
"""
from fabric.api import *
from fabric.main import main
from fabric.colors import *
from fabric.context_managers import *
from fabric.contrib.console import confirm
import os
import sys
import re
import getpass

config = {
"deploy_to": '/var/www/my_app_name',
"scm": 'git',
"repo_url": 'https://github.com/DingGuodong/GoogleHostsFileForLinux.git',
"branch": 'master',
"log_level": 'debug',
"keep_releases": 10
}

env.roledefs = {
'test': ['root@10.6.28.28:22', ],
'nginx': ['root@10.6.28.46:22', 'root@10.6.28.27:22', ],
'db': ['root@10.6.28.35:22', 'root@10.6.28.93:22', ],
'sit': ['root@10.6.28.46:22', 'root@10.6.28.135:22', 'root@10.6.28.35:22', ],
'uat': ['root@10.6.28.27:22', 'root@10.6.28.125:22', 'root@10.6.28.93:22', ],
'all': ["10.6.28.27", "10.6.28.28", "10.6.28.35", "10.6.28.46", "10.6.28.93", "10.6.28.125", "10.6.28.135"]
}

env.user = "root"
env.hosts = ["10.6.28.27", "10.6.28.28", "10.6.28.35", "10.6.28.46", "10.6.28.93", "10.6.28.125", "10.6.28.135"]

def win_or_linux():
# os.name ->(sames to) sys.builtin_module_names
if 'posix' in sys.builtin_module_names:
os_type = 'Linux'
elif 'nt' in sys.builtin_module_names:
os_type = 'Windows'
return os_type

def is_windows():
if "windows" in win_or_linux().lower():
return True
else:
return False

def is_linux():
if "linux" in win_or_linux().lower():
return True
else:
return False

class Capistrano(object):
class SCM(object):
class Git(object):
def __init__(self):
self.repo_url = None
self.name = None
self.branch = None
self.repo_path = None
self.user = None

def set(self, repo_url, branch=None, repo_path=None):
if repo_url is None:
abort("You must specify a repository to clone.")
else:
self.repo_url = repo_url
if branch is None:
self.branch = "master"
else:
self.branch = branch

pattern = re.compile(r"(\w+)(?=\.git$)")
match = pattern.search(repo_url)
if match:
paths = match.group()
else:
paths = None
if repo_path is not None and not os.path.exists(repo_path):
try:
os.mkdir(repo_path)
except IOError:
repo_path = os.path.join(os.path.dirname(__file__), paths)
elif repo_path is None:
repo_path = ""
self.repo_path = os.path.abspath(repo_path)

def clone(self):
local("git clone --branch %s %s %s" % (self.branch, self.repo_url, self.repo_path))

def check(self):
with lcd(self.repo_path):
return local("git ls-remote --heads %s" % self.repo_url, capture=True)

def pull(self):
with lcd(self.repo_path):
if os.path.exists(os.path.join(self.repo_path, ".git")):
local("git pull origin %s" % self.branch)
else:
self.clone()
self.pull()

def update(self):
pass

def status(self):
with lcd(self.repo_path):
local("git status")

def branch(self):
with lcd(self.repo_path):
local("git rev-parse --abbrev-ref HEAD", capture=True)

def long_id(self):
with lcd(self.repo_path):
return local("git rev-parse HEAD", capture=True)

def short_id(self):
with lcd(self.repo_path):
return local("git rev-parse --short HEAD", capture=True)

def fetch_revision(self):
with lcd(self.repo_path):
return local("git rev-list --max-count=1 %s" % self.branch, capture=True)

def user(self):
if is_linux():
self.user = "%s(%s)" % (os.getlogin(), os.getuid())
if is_windows():
import getpass
self.user = getpass.getuser()

class DSL(object):
class Paths(object):
def __init__(self):
self.deploy_to = config['deploy_to']
self.current = None

# TODO(Guodong Ding) fetch 'deploy_to' from config file or dict
def deploy_path(self):
return os.path.abspath(self.deploy_to)

def current_path(self):
current_directory = "current"
return os.path.join(self.deploy_path(), current_directory)

def releases_path(self):
return os.path.join(self.deploy_path(), "releases")

def set_release_path(self):
import datetime
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
self.current = os.path.join(self.releases_path(), timestamp)
return os.path.join(self.releases_path(), timestamp)

def shared_path(self):
return os.path.join(self.deploy_path(), "shared")

def repo_path(self):
return os.path.join(self.deploy_path(), "repo")

def revision_log(self):
return os.path.join(self.deploy_path(), "revisions.log")

def __paths(self):
return self.releases_path(), self.repo_path(), self.shared_path()

def makepaths(self):
for directory in self.__paths():
if not os.path.exists(directory):
os.makedirs(directory)

def make_release_dirs(self):
os.makedirs(self.set_release_path())

def make_current(self):
if is_linux():
if os.path.exists(self.current_path()) and os.path.islink(self.current_path()):
os.unlink(self.current_path())
os.symlink(self.current, self.current_path())
if is_windows():
if os.path.exists(self.current_path()):
import shutil
shutil.rmtree(self.current_path())
try:
local("ln -sd %s %s" % (self.current, self.current_path()))
except Exception:
raise NotImplementedError

def update_revision_log(self, branch=None, sid=None, release=None, by=None):
print blue("Log details of the deploy")
with open(self.revision_log(), 'a') as f:
f.write("Branch %s (at %s) deployed as release %s by %s\n" % (branch, sid, release, by))

def cleanup(self):
keep_releases = config['keep_releases']
releases = local("ls -xtr %s" % self.releases_path(), capture=True).split()
# print releases[-keep_releases:]
if len(releases) > keep_releases:
for release in releases[0:(len(releases) - keep_releases)]:
local("rm -rf %s" % os.path.join(self.releases_path(), release))

@staticmethod
def __get_file_last_line(inputfile):
filesize = os.path.getsize(inputfile)
blocksize = 1024
with open(inputfile, 'rb') as f:
last_line = ""
if filesize > blocksize:
maxseekpoint = (filesize // blocksize)
f.seek((maxseekpoint - 1) * blocksize)
elif filesize:
f.seek(0, 0)
lines = f.readlines()
if lines:
lineno = 1
while last_line == "":
last_line = lines[-lineno].strip()
lineno += 1
return last_line

def rollback(self):
print blue("Revert to previous release timestamp")
revision_log_message = self.__get_file_last_line(self.revision_log())
last_release = None
import re
s = re.compile(r"release (.*) by")
match = s.search(revision_log_message)
if match:
last_release = match.groups()[0]
else:
abort("Can NOT found rollback release in revision log files, %s." % self.revision_log())
if os.path.exists(last_release):
print yellow("Symlink previous release to current")
else:
abort("Can NOT found rollback release on filesystem.")
if is_linux():
if os.path.exists(self.current_path()) and os.path.islink(self.current_path()):
os.unlink(self.current_path())
os.symlink(last_release, self.current_path())
if is_windows():
if os.path.exists(self.current_path()):
import shutil
shutil.rmtree(self.current_path())
try:
local("ln -sd %s %s" % (last_release, self.current_path()))
except Exception:
raise NotImplementedError

class Application(object):
class Deploy(object):
def __init__(self):
self.P = Capistrano.DSL.Paths()
self.G = Capistrano.SCM.Git()

def deploy(self):
# TODO(Guodong Ding): core job here, this is a deploy demo
with lcd(self.P.current_path()):
try:
src = os.path.join(self.P.repo_path(), "replaceLocalHostsFileAgainstGfw.sh")
local_path = os.path.join(self.P.current_path(), "hosts")
remote_path = "/tmp/replaceLocalHostsFileAgainstGfw.sh"
with open(src, 'r') as f:
content = f.read()
with open(local_path, "w") as f:
f.write(content)
if os.path.getsize(local_path):
print red("upload files to remote hosts")
put(local_path, remote_path)
run("chmod +x %s" % remote_path)
run("ls -al %s" % remote_path)
print red("deploy test demo successfully!")
except IOError:
raise NotImplementedError

def run(self):
print blue("Do deploy procedure.")
self.P.makepaths()
self.G.set(config["repo_url"], repo_path=self.P.repo_path())
self.G.pull()

self.P.make_release_dirs()
self.P.make_current()
self.deploy()
self.P.update_revision_log(self.G.branch, self.G.short_id(), self.P.current, getpass.getuser())
self.P.cleanup()
print green("Deploy successfully!")

@roles("test")
def test_deploy():
c = Capistrano.Application.Deploy()
c.run()

def terminal_debug(defName):
command = "fab -i c:\Users\Guodong\.ssh\exportedkey201310171355\
-f %s \
%s" % (__file__, defName)
os.system(command)
sys.exit(0)

if __name__ == '__main__':
if len(sys.argv) == 1 and is_windows():
terminal_debug("test_deploy")

sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
print red("Please use 'fab -f %s'" % " ".join(str(x) for x in sys.argv[0:]))
sys.exit(1)

关于Fabric的进一步使用可以参考GitHub上的另一个文件：https://github.com/DingGuodong/LinuxBashShellScriptForOps/blob/master/projects/autoOps/pythonSelf/fabfile.py 此文件有更多关于Fabric的示例，可以更好的说明Fabric的用途。
最后，不得不说Python是优秀的编程、脚本语言，用在运维上确实很方便，只需短短的几天时间就可以编写出有用的脚本。作为运维人员不必排斥编程，编程是为了更好的运维。如果觉得本文有用，可以继续关注这个GitHub项目（https://github.com/DingGuodong/LinuxBashShellScriptForOps），这个项目会持续完善，积累更多有用的Shell、Python编程和运维的相关知识和文件。
--end--