Linux多线程下载命令axel编译安装 支持断点续传 Yum加速改造

Linux多线程下载命令axel编译安装

wget 'http://www.ha97.com/code/axel-2.4.tar.gz'
# 找到一个还可以下载的网址
# ubuntu可以直接使用apt-get下载使用

tar -xf axel-2.4.tar.gz
cd axel-2.4
yum -y install gcc
./configure
make && make install
# 编译安装，需要gcc环境

axel -h
# 查看帮助

axel -n 5 -s 1024 'https://mirrors.tuna.tsinghua.edu.cn/mariadb//mariadb-10.2.12/source/mariadb-10.2.12.tar.gz'
# 测试下载
# 使用5个线程同时下载文件
# 限定下载速度上限为1024B/S
# 可以ctrl+c停掉，再次执行
# 发现可以断点续传

Yum加速改造

sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf

cd /etc/yum/pluginconf.d/
cat >axelget.conf<<EOF
[main]
enabled=1
onlyhttp=1
enablesize=54000
cleanOnException=1
EOF

cd /usr/lib/yum-plugins/
cat >axelget.py<<EOF
from yum.plugins import PluginYumExit, TYPE_CORE, TYPE_INTERACTIVE
from urlparse import urljoin
import os,time

requires_api_version = '2.3'
plugin_type = (TYPE_CORE, TYPE_INTERACTIVE)

enablesize=300000
trymirrornum=-1
maxconn=10
httpdownloadonly=False
cleanOnException=0

def init_hook(conduit):
global enablesize,trymirrornum,maxconn,cleanOnException,httpdownloadonly
enablesize = conduit.confInt('main','enablesize',default=30000)
trymirrornum = conduit.confInt('main','trymirrornum',default=-1)
maxconn = conduit.confInt('main','maxconn',default=10)
httpdownloadonly=conduit.confBool('main','onlyhttp',default=False)
cleanOnException=conduit.confInt('main','cleanOnException',default=0)
return

def predownload_hook(conduit):
global enablesize,cleanOnException,httpdownloadonly
preffermirror=""
PkgIdx=0
TotalPkg=len(conduit.getDownloadPackages())
for po in (conduit.getDownloadPackages()):
PkgIdx+=1
if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
continue
totsize = long(po.size)
ret = False
if totsize <= enablesize:
conduit.info(2, "Package %s download size %d less than %d,Skip plugin!"  % (po.repo.id,totsize,enablesize))
continue
else:
conduit.info(2, "[%d/%d]Ok,we will try to use axel to download this big file:%d" % (PkgIdx,TotalPkg,totsize))
local = po.localPkg()
if os.path.exists(local):
if not os.path.exists(local+".st"):
fstate=os.stat(local)
if totsize == fstate.st_size:
conduit.info(2,"Target already exists,skip to next file!")
continue
localall = "%s %s" % (local,local+".st")
rmcmd = "rm -f %s" % (localall)
curmirroridx = 0
conduit.info(2,"Before we start,clean all the key files")
os.system(rmcmd)
connnum = totsize / enablesize
if connnum*enablesize<totsize:
connnum+=1
if connnum > maxconn:
connnum = maxconn
mirrors=[]
mirrors[:0]=po.repo.urls
if preffermirror != "":
mirrors[:0] = [preffermirror]
for url in mirrors:
if url.startswith("ftp://") and httpdownloadonly:
print "Skip Ftp Site:",url
continue
if url.startswith("file://"):
print "Skip Local Site:",url
continue
curmirroridx += 1
if (curmirroridx > trymirrornum) and (trymirrornum != -1):
conduit.info(2, "Package %s has tried %d mirrors,Skip plugin!" % (po.repo.id,trymirrornum))
break
remoteurl =  "%s/%s" % (url,po.remote_path)
syscmd = "axel -a -n %s %s -o %s" % (connnum,remoteurl,local)
conduit.info(2, "Execute axel cmd:\n%s"  % syscmd)
os.system(syscmd)
time.sleep(2)
if os.path.exists(local+".st"):
conduit.info(2,"axel exit by exception,let's try another mirror")
if cleanOnException:
conduit.i
4000
nfo(2,"because cleanOnException is set to 1,we do remove key file first")
os.system(rmcmd)
continue
elif not os.path.exists(local):#this mirror may not update yet
continue
else:
ret = True
preffermirror=url
break
if not ret:
conduit.info (2,"try to run rm cmd:%s"  % rmcmd)
os.system(rmcmd)
EOF

yum -y install zip unzip
# 测试，观察日志是否使用axel进行下载