From 9640e792e1ac30e41f08e1954d56ad11398c48f1 Mon Sep 17 00:00:00 2001 From: "guangjun.hgj" Date: Fri, 24 May 2019 11:30:35 +0800 Subject: [PATCH] Release 1.80.6 --- ChangeLog | 492 +++++++++++++++++++------------------ configure.ac | 570 +++++++++++++++++++++---------------------- scripts/build-pkg.py | 544 +++++++++++++++++++++-------------------- 3 files changed, 806 insertions(+), 800 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5857534..41739a1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,244 +1,248 @@ -ChangeLog for OSSFS ------------------- -## v1.80.5 (25/05/2018) - -- Lock FdEntity when race condition - -## v1.80.4 (11/04/2018) - -- fix request hang when flush the writing file -- update acl description of user in README -- sleep for awhile when curl_multi_fdset failed - -## v1.80.3 (12/10/2017) - -- Fix a rename bug - -## v1.80.2 (06/01/2017) - -- Fix list error when path contains multi-slash -- Print verbose message when failed to mount ossfs - -## v1.80.1 (05/16/2017) - -- Fix noobj cache bug -- Fix null path in flush - -## v1.80.0 (12/14/2016) - -- Relax dir object content-type (#34) - -## v1.79.9 (6/17/2016) - -- Print to stderr when check OSS fails (#31) - -## v1.79.8 (4/22/2016) - -- Fix a bug when truncating an empty file (#27) - -## v1.79.7 (4/19/2016) - -- Change object default permission to 777 -- Add curl handler pool to reuse connections (#24) - -## v1.79.6 (3/29/2016) - -- Add the missing User-Agent in multi-head request - -## v1.79.5 (3/29/2016) - -- Setup travis CI -- Fix a bug in fd cache lookup -- Add script to build dist package - -## v1.79.4 (3/21/2016) - -- Add noxattr option to disable get/set xattr - -ChangeLog for S3FS ------------------- - -Version 1.79 -- Jul 19, 2015 -issue #60 - Emit user-friendly log messages on failed CheckBucket requests -issue #62 - Remove stray chars from source files -issue #63 - Fix spelling errors -issue #68 - FreeBSD issue -issue #69 - Address clang always true warnings -issue #73 - Small gitignore fixes -issue #74 - url: handle scheme omission -issue #83 - Changed option processing to use strtol() to get a umask -issue #93 - Add simple unit tests for trim functions -issue #100 - CURL handles not properly initialized to use DNS or SSL session caching -issue #101 - Optimized function "bool directory_empty()" -issue #103 - Remove prefix option in s3fs man page - issue#87 -issue #104 - fix rename before close -issue #116 - Supported signature version 4 -issue #119 - Added new mp_umask option about issue#107, pr#110 -issue #124 - Fallback to v2 signatures correctly. -issue #130 - refactor integration tests create/cleanup file -issue #131 - Test ls -issue #132 - Use S3Proxy to run integration tests -issue #134 - Include Content-Type in complete MPU V2 signature -issue #135 - Correct V4 signature for initiate multipart upload -issue #136 - Small fixes to integration tests -issue #137 - Add test for multi-part upload -issue #138 - Fixed bugs, not turn use_cache off and ty to load to end - issue#97 -issue #143 - Fixed a bug no use_cache case about fixed #138 - issue#141 -issue #144 - Add Travis configuration -issue #146 - add exit handler to cleanup on failures -issue #147 - Use S3Proxy 1.4.0-SNAPSHOT -issue #150 - Fixed a bug not handling fsync - #145 -issue #154 - Fixed url-encoding for ampersand etc on sigv4 - Improvement/#149 -issue #155 - Fixed a bug: unable to mount bucket subdirectory -issue #156 - Fixed a bug about ssl session sharing with libcurl older 7.23.0 - issue#126 -issue #159 - Upgrade to S3Proxy 1.4.0 -issue #164 - send the correct Host header when using -o url -issue #165 - Auth v4 refactor -issue #167 - Increased default connecting/reading/writing timeout value -issue #168 - switch to use region specific endpoints to compute correct v4 signature -issue #170 - Reviewed and fixed response codes print in curl.cpp - #157 -issue #171 - Support buckets with mixed-case names -issue #173 - Run integration tests via Travis -issue #176 - configure.ac: detect target, if target is darwin (OSX), then #176 -issue #177 - Add .mailmap -issue #178 - Update .gitignore -issue #184 - Add usage information for multipart_size -issue #185 - Correct obvious typos in usage and README -issue #190 - Add a no_check_certificate option. -issue #194 - Tilda in a file-name breaks things (EPERM) -issue #198 - Disasble integration tests for Travis -issue #199 - Supported extended attributes(retry) -issue #200 - fixed fallback to sigv2 for bucket create and GCS -issue #202 - Specialize {set,get}xattr for OS X -issue #204 - Add integration test for xattr -issue #207 - Fixed a few small spelling issues. - -Version 1.78 -- Sep 15, 2014 -issue #29 - Possible to create Debian/Ubuntu packages?(googlecode issue 109) -issue 417(googlecode) - Password file with DOS format is not handled properly -issue #41 - Failed making signature -issue #40 - Moving a directory containing more than 1000 files truncates the directory -issue #49 - use_sse is ignored when creating new files -issue #39 - Support for SSE-C -issue #50 - Cannot find pkg-config when configured with any SSL backend except openssl - -Version 1.77 -- Apr 19, 2014 -issue 405(googlecode) - enable_content_md5 Input/output error -issue #14 - s3fs -u should return 0 if there are no lost multiparts -issue #16 - empty file is written to s3 -issue #18 - s3fs crashes with segfault -issue #22 - Fix typos in docs for max_stat_cache_size -issue #23 - curl ssl problems -issue #28 - Address signedness warning in FdCache::Init - -Version 1.76 -- Jan 21, 2014 -issue #5 - du shows incorrect usage stats -issue #8 - version in configure.ac is 1.74 for release 1.75 - -Version 1.75 -- Jan 6, 2014 -issue #1 - Using %20 instead of the plus (+) sign for encoding spaces -issue #3 - Fixed local timezone was incorrectly being applied to RAM and Last-Modified dates. -issue #4 - Fix compilation error on MacOSX with missing const - -Version 1.74 -- Nov 24, 2013 -This version is initial version on Github, same as on GoogleCodes(s3fs). -https://github.com/s3fs-fuse/s3fs-fuse/releases/tag/v1.74 -see more detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.74.tar.gz - -Version 1.73 -- Aug 23, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.73.tar.gz - -Version 1.72 -- Aug 10, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.72.tar.gz - -Version 1.71 -- Jun 15, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.71.tar.gz - -Version 1.70 -- Jun 01, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.70.tar.gz - -Version 1.69 -- May 15, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.69.tar.gz - -Version 1.68 -- Apr 30, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.68.tar.gz - -Version 1.67 -- Apr 13, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.67.tar.gz - -Version 1.66 -- Apr 06, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.66.tar.gz - -Version 1.65 -- Mar 30, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.65.tar.gz - -Version 1.64 -- Mar 23, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.64.tar.gz - -Version 1.63 -- Feb 24, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.63.tar.gz - -Version 1.62 -- Jan 27, 2013 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.62.tar.gz - -Version 1.61 -- Aug 30, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.61.tar.gz - -Version 1.60 -- Aug 29, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.60.tar.gz - -Version 1.59 -- Jul 28, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.59.tar.gz - -Version 1.58 -- Jul 19, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.58.tar.gz - -Version 1.57 -- Jul 07, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.57.tar.gz - -Version 1.56 -- Jul 07, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.56.tar.gz - -Version 1.55 -- Jul 02, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.55.tar.gz - -Version 1.54 -- Jun 25, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.54.tar.gz - -Version 1.53 -- Jun 22, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.53.tar.gz - -Version 1.40 -- Feb 11, 2011 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.40.tar.gz - -Version 1.33 -- Dec 30, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.33.tar.gz - -Version 1.25 -- Dec 16, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.25.tar.gz - -Version 1.19 -- Dec 2, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.19.tar.gz - -Version 1.16 -- Nov 22, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.16.tar.gz - -Version 1.10 -- Nov 6, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.10.tar.gz - -Version 1.02 -- Oct 29, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.02.tar.gz - -Version 1.01 -- Oct 28, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.01.tar.gz - -Version 1.0 -- Oct 24, 2010 -see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.0.tar.gz - ------- -Version 1.1 -- Mon Oct 18 2010 -Dan Moore reopens the project and fixes various issues that had accumulated in the tracker. Adrian Petrescu converts the project to autotools and posts it to GitHub. - -Version 1.0 -- 2008 -Randy Rizun releases a basic version of S3FS on Google Code. - +ChangeLog for OSSFS +------------------ +## v1.80.6 (24/05/2019) + +- support ECSRole + +## v1.80.5 (25/05/2018) + +- Lock FdEntity when race condition + +## v1.80.4 (11/04/2018) + +- fix request hang when flush the writing file +- update acl description of user in README +- sleep for awhile when curl_multi_fdset failed + +## v1.80.3 (12/10/2017) + +- Fix a rename bug + +## v1.80.2 (06/01/2017) + +- Fix list error when path contains multi-slash +- Print verbose message when failed to mount ossfs + +## v1.80.1 (05/16/2017) + +- Fix noobj cache bug +- Fix null path in flush + +## v1.80.0 (12/14/2016) + +- Relax dir object content-type (#34) + +## v1.79.9 (6/17/2016) + +- Print to stderr when check OSS fails (#31) + +## v1.79.8 (4/22/2016) + +- Fix a bug when truncating an empty file (#27) + +## v1.79.7 (4/19/2016) + +- Change object default permission to 777 +- Add curl handler pool to reuse connections (#24) + +## v1.79.6 (3/29/2016) + +- Add the missing User-Agent in multi-head request + +## v1.79.5 (3/29/2016) + +- Setup travis CI +- Fix a bug in fd cache lookup +- Add script to build dist package + +## v1.79.4 (3/21/2016) + +- Add noxattr option to disable get/set xattr + +ChangeLog for S3FS +------------------ + +Version 1.79 -- Jul 19, 2015 +issue #60 - Emit user-friendly log messages on failed CheckBucket requests +issue #62 - Remove stray chars from source files +issue #63 - Fix spelling errors +issue #68 - FreeBSD issue +issue #69 - Address clang always true warnings +issue #73 - Small gitignore fixes +issue #74 - url: handle scheme omission +issue #83 - Changed option processing to use strtol() to get a umask +issue #93 - Add simple unit tests for trim functions +issue #100 - CURL handles not properly initialized to use DNS or SSL session caching +issue #101 - Optimized function "bool directory_empty()" +issue #103 - Remove prefix option in s3fs man page - issue#87 +issue #104 - fix rename before close +issue #116 - Supported signature version 4 +issue #119 - Added new mp_umask option about issue#107, pr#110 +issue #124 - Fallback to v2 signatures correctly. +issue #130 - refactor integration tests create/cleanup file +issue #131 - Test ls +issue #132 - Use S3Proxy to run integration tests +issue #134 - Include Content-Type in complete MPU V2 signature +issue #135 - Correct V4 signature for initiate multipart upload +issue #136 - Small fixes to integration tests +issue #137 - Add test for multi-part upload +issue #138 - Fixed bugs, not turn use_cache off and ty to load to end - issue#97 +issue #143 - Fixed a bug no use_cache case about fixed #138 - issue#141 +issue #144 - Add Travis configuration +issue #146 - add exit handler to cleanup on failures +issue #147 - Use S3Proxy 1.4.0-SNAPSHOT +issue #150 - Fixed a bug not handling fsync - #145 +issue #154 - Fixed url-encoding for ampersand etc on sigv4 - Improvement/#149 +issue #155 - Fixed a bug: unable to mount bucket subdirectory +issue #156 - Fixed a bug about ssl session sharing with libcurl older 7.23.0 - issue#126 +issue #159 - Upgrade to S3Proxy 1.4.0 +issue #164 - send the correct Host header when using -o url +issue #165 - Auth v4 refactor +issue #167 - Increased default connecting/reading/writing timeout value +issue #168 - switch to use region specific endpoints to compute correct v4 signature +issue #170 - Reviewed and fixed response codes print in curl.cpp - #157 +issue #171 - Support buckets with mixed-case names +issue #173 - Run integration tests via Travis +issue #176 - configure.ac: detect target, if target is darwin (OSX), then #176 +issue #177 - Add .mailmap +issue #178 - Update .gitignore +issue #184 - Add usage information for multipart_size +issue #185 - Correct obvious typos in usage and README +issue #190 - Add a no_check_certificate option. +issue #194 - Tilda in a file-name breaks things (EPERM) +issue #198 - Disasble integration tests for Travis +issue #199 - Supported extended attributes(retry) +issue #200 - fixed fallback to sigv2 for bucket create and GCS +issue #202 - Specialize {set,get}xattr for OS X +issue #204 - Add integration test for xattr +issue #207 - Fixed a few small spelling issues. + +Version 1.78 -- Sep 15, 2014 +issue #29 - Possible to create Debian/Ubuntu packages?(googlecode issue 109) +issue 417(googlecode) - Password file with DOS format is not handled properly +issue #41 - Failed making signature +issue #40 - Moving a directory containing more than 1000 files truncates the directory +issue #49 - use_sse is ignored when creating new files +issue #39 - Support for SSE-C +issue #50 - Cannot find pkg-config when configured with any SSL backend except openssl + +Version 1.77 -- Apr 19, 2014 +issue 405(googlecode) - enable_content_md5 Input/output error +issue #14 - s3fs -u should return 0 if there are no lost multiparts +issue #16 - empty file is written to s3 +issue #18 - s3fs crashes with segfault +issue #22 - Fix typos in docs for max_stat_cache_size +issue #23 - curl ssl problems +issue #28 - Address signedness warning in FdCache::Init + +Version 1.76 -- Jan 21, 2014 +issue #5 - du shows incorrect usage stats +issue #8 - version in configure.ac is 1.74 for release 1.75 + +Version 1.75 -- Jan 6, 2014 +issue #1 - Using %20 instead of the plus (+) sign for encoding spaces +issue #3 - Fixed local timezone was incorrectly being applied to RAM and Last-Modified dates. +issue #4 - Fix compilation error on MacOSX with missing const + +Version 1.74 -- Nov 24, 2013 +This version is initial version on Github, same as on GoogleCodes(s3fs). +https://github.com/s3fs-fuse/s3fs-fuse/releases/tag/v1.74 +see more detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.74.tar.gz + +Version 1.73 -- Aug 23, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.73.tar.gz + +Version 1.72 -- Aug 10, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.72.tar.gz + +Version 1.71 -- Jun 15, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.71.tar.gz + +Version 1.70 -- Jun 01, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.70.tar.gz + +Version 1.69 -- May 15, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.69.tar.gz + +Version 1.68 -- Apr 30, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.68.tar.gz + +Version 1.67 -- Apr 13, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.67.tar.gz + +Version 1.66 -- Apr 06, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.66.tar.gz + +Version 1.65 -- Mar 30, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.65.tar.gz + +Version 1.64 -- Mar 23, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.64.tar.gz + +Version 1.63 -- Feb 24, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.63.tar.gz + +Version 1.62 -- Jan 27, 2013 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.62.tar.gz + +Version 1.61 -- Aug 30, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.61.tar.gz + +Version 1.60 -- Aug 29, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.60.tar.gz + +Version 1.59 -- Jul 28, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.59.tar.gz + +Version 1.58 -- Jul 19, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.58.tar.gz + +Version 1.57 -- Jul 07, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.57.tar.gz + +Version 1.56 -- Jul 07, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.56.tar.gz + +Version 1.55 -- Jul 02, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.55.tar.gz + +Version 1.54 -- Jun 25, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.54.tar.gz + +Version 1.53 -- Jun 22, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.53.tar.gz + +Version 1.40 -- Feb 11, 2011 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.40.tar.gz + +Version 1.33 -- Dec 30, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.33.tar.gz + +Version 1.25 -- Dec 16, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.25.tar.gz + +Version 1.19 -- Dec 2, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.19.tar.gz + +Version 1.16 -- Nov 22, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.16.tar.gz + +Version 1.10 -- Nov 6, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.10.tar.gz + +Version 1.02 -- Oct 29, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.02.tar.gz + +Version 1.01 -- Oct 28, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.01.tar.gz + +Version 1.0 -- Oct 24, 2010 +see detail on googlecodes: http://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.0.tar.gz + +------ +Version 1.1 -- Mon Oct 18 2010 +Dan Moore reopens the project and fixes various issues that had accumulated in the tracker. Adrian Petrescu converts the project to autotools and posts it to GitHub. + +Version 1.0 -- 2008 +Randy Rizun releases a basic version of S3FS on Google Code. + diff --git a/configure.ac b/configure.ac index 2e173f6..bc74429 100644 --- a/configure.ac +++ b/configure.ac @@ -1,285 +1,285 @@ -###################################################################### -# ossfs - FUSE-based file system backed by Aliyun OSS -# -# Copyright 2007-2008 Randy Rizun -# Copyright 2015 Haoran Yang -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -###################################################################### -dnl Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(ossfs, 1.80.5) -AC_CONFIG_HEADER([config.h]) - -AC_CANONICAL_SYSTEM -AM_INIT_AUTOMAKE([foreign]) - -AC_PROG_CXX -AC_PROG_CC - -CXXFLAGS="$CXXFLAGS -Wall -D_FILE_OFFSET_BITS=64" - -dnl ---------------------------------------------- -dnl For OSX -dnl ---------------------------------------------- -case "$target" in - *-darwin* ) - # Do something specific for mac - min_fuse_version=2.7.3 - ;; - *) - # Default Case - # assume other supported linux system - min_fuse_version=2.8.4 - ;; -esac - -dnl ---------------------------------------------- -dnl Choice SSL library -dnl ---------------------------------------------- -auth_lib=na -nettle_lib=no - -dnl -dnl nettle library -dnl -AC_MSG_CHECKING([ossfs build with nettle(GnuTLS)]) -AC_ARG_WITH( - nettle, - [AS_HELP_STRING([--with-nettle], [ossfs build with nettle in GnuTLS(default no)])], - [ - case "${withval}" in - yes) - AC_MSG_RESULT(yes) - nettle_lib=yes - ;; - *) - AC_MSG_RESULT(no) - ;; - esac - ], - [ - AC_MSG_RESULT(no) - ]) - -dnl -dnl use openssl library for ssl -dnl -AC_MSG_CHECKING([ossfs build with OpenSSL]) -AC_ARG_WITH( - openssl, - [AS_HELP_STRING([--with-openssl], [ossfs build with OpenSSL(default is no)])], - [ - case "${withval}" in - yes) - AC_MSG_RESULT(yes) - AS_IF( - [test $nettle_lib = no], - [auth_lib=openssl], - [AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])]) - ;; - *) - AC_MSG_RESULT(no) - ;; - esac - ], - [ - AC_MSG_RESULT(no) - ]) - -dnl -dnl use GnuTLS library for ssl -dnl -AC_MSG_CHECKING([ossfs build with GnuTLS]) -AC_ARG_WITH( - gnutls, - [AS_HELP_STRING([--with-gnutls], [ossfs build with GnuTLS(default is no)])], - [ - case "${withval}" in - yes) - AC_MSG_RESULT(yes) - AS_IF( - [test $auth_lib = na], - [ - AS_IF( - [test $nettle_lib = no], - [auth_lib=gnutls], - [auth_lib=nettle]) - ], - [AC_MSG_ERROR([could not set gnutls because already set another ssl library])]) - ;; - *) - AC_MSG_RESULT(no) - ;; - esac - ], - [ - AC_MSG_RESULT(no) - ]) - -dnl -dnl use nss library for ssl -dnl -AC_MSG_CHECKING([ossfs build with NSS]) -AC_ARG_WITH( - nss, - [AS_HELP_STRING([--with-nss], [ossfs build with NSS(default is no)])], - [ - case "${withval}" in - yes) - AC_MSG_RESULT(yes) - AS_IF( - [test $auth_lib = na], - [ - AS_IF( - [test $nettle_lib = no], - [auth_lib=nss], - [AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])]) - ], - [AC_MSG_ERROR([could not set nss because already set another ssl library])]) - ;; - *) - AC_MSG_RESULT(no) - ;; - esac - ], - [ - AC_MSG_RESULT(no) - ]) - -AS_IF( - [test $auth_lib = na], - AS_IF( - [test $nettle_lib = no], - [auth_lib=openssl], - [AC_MSG_ERROR([could not set nettle without GnuTLS library])] - ) -) - -dnl -dnl For PKG_CONFIG before checking nss/gnutls. -dnl this is redundant checking, but we need checking before following. -dnl -PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6]) - -AC_MSG_CHECKING([compile ossfs with]) -case "${auth_lib}" in -openssl) - AC_MSG_RESULT(OpenSSL) - PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9]) - ;; -gnutls) - AC_MSG_RESULT(GnuTLS-gcrypt) - gnutls_nettle="" - AC_CHECK_LIB(gnutls, gcry_control, [gnutls_nettle=0]) - AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])]) - AS_IF([test $gnutls_nettle = 0], - [ - PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ]) - LIBS="-lgnutls -lgcrypt $LIBS" - AC_MSG_CHECKING([gnutls is build with]) - AC_MSG_RESULT(gcrypt) - ], - [AC_MSG_ERROR([GnuTLS found, but gcrypt not found])]) - ;; -nettle) - AC_MSG_RESULT(GnuTLS-nettle) - gnutls_nettle="" - AC_CHECK_LIB(gnutls, nettle_MD5Init, [gnutls_nettle=1]) - AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])]) - AS_IF([test $gnutls_nettle = 1], - [ - PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ]) - LIBS="-lgnutls -lnettle $LIBS" - AC_MSG_CHECKING([gnutls is build with]) - AC_MSG_RESULT(nettle) - ], - [AC_MSG_ERROR([GnuTLS found, but nettle not found])]) - ;; -nss) - AC_MSG_RESULT(NSS) - PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ]) - ;; -*) - AC_MSG_ERROR([unknown ssl library type.]) - ;; -esac - -AM_CONDITIONAL([USE_SSL_OPENSSL], [test "$auth_lib" = openssl]) -AM_CONDITIONAL([USE_SSL_GNUTLS], [test "$auth_lib" = gnutls -o "$auth_lib" = nettle]) -AM_CONDITIONAL([USE_GNUTLS_NETTLE], [test "$auth_lib" = nettle]) -AM_CONDITIONAL([USE_SSL_NSS], [test "$auth_lib" = nss]) - -dnl ---------------------------------------------- -dnl check functions -dnl ---------------------------------------------- -dnl malloc_trim function -AC_CHECK_FUNCS([malloc_trim]) - -dnl ---------------------------------------------- -dnl check symbols/macros/enums -dnl ---------------------------------------------- -dnl PTHREAD_MUTEX_RECURSIVE -AC_MSG_CHECKING([pthread mutex recursive]) -AC_COMPILE_IFELSE( - [AC_LANG_PROGRAM([[#include ]], - [[int i = PTHREAD_MUTEX_RECURSIVE;]]) - ], - [AC_DEFINE(S3FS_MUTEX_RECURSIVE, PTHREAD_MUTEX_RECURSIVE, [Define if you have PTHREAD_MUTEX_RECURSIVE]) - AC_MSG_RESULT(PTHREAD_MUTEX_RECURSIVE) - ], - [AC_COMPILE_IFELSE( - [AC_LANG_PROGRAM([[#include ]], - [[int i = PTHREAD_MUTEX_RECURSIVE_NP;]]) - ], - [AC_DEFINE(S3FS_MUTEX_RECURSIVE, PTHREAD_MUTEX_RECURSIVE_NP, [Define if you have PTHREAD_MUTEX_RECURSIVE_NP]) - AC_MSG_RESULT(PTHREAD_MUTEX_RECURSIVE_NP) - ], - [AC_MSG_ERROR([do not have PTHREAD_MUTEX_RECURSIVE symbol])]) - ] -) - -dnl ---------------------------------------------- -dnl output files -dnl ---------------------------------------------- -AC_CONFIG_FILES(Makefile src/Makefile test/Makefile doc/Makefile) - -dnl ---------------------------------------------- -dnl short commit hash -dnl ---------------------------------------------- -AC_CHECK_PROG([GITCMD], [git —version], [yes], [no]) -AC_CHECK_FILE([.git], [DOTGITDIR=yes], [DOTGITDIR=no]) - -AC_MSG_CHECKING([github short commit hash]) -if test “x${GITCMD}” = “xyes” -a “x${DOTGITDIR}” = “xyes”; then - GITCOMMITHASH=`git rev-parse --short HEAD` -elif test -f default_commit_hash; then - GITCOMMITHASH=`cat default_commit_hash` -else - GITCOMMITHASH="unknown" -fi -AC_MSG_RESULT([${GITCOMMITHASH}]) - -AC_DEFINE_UNQUOTED([COMMIT_HASH_VAL], ["${GITCOMMITHASH}"], [short commit hash value on github]) - -dnl ---------------------------------------------- -dnl put -dnl ---------------------------------------------- -AC_OUTPUT - -dnl ---------------------------------------------- -dnl end configuration -dnl ---------------------------------------------- +###################################################################### +# ossfs - FUSE-based file system backed by Aliyun OSS +# +# Copyright 2007-2008 Randy Rizun +# Copyright 2015 Haoran Yang +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +###################################################################### +dnl Process this file with autoconf to produce a configure script. + +AC_PREREQ(2.59) +AC_INIT(ossfs, 1.80.6) +AC_CONFIG_HEADER([config.h]) + +AC_CANONICAL_SYSTEM +AM_INIT_AUTOMAKE([foreign]) + +AC_PROG_CXX +AC_PROG_CC + +CXXFLAGS="$CXXFLAGS -Wall -D_FILE_OFFSET_BITS=64" + +dnl ---------------------------------------------- +dnl For OSX +dnl ---------------------------------------------- +case "$target" in + *-darwin* ) + # Do something specific for mac + min_fuse_version=2.7.3 + ;; + *) + # Default Case + # assume other supported linux system + min_fuse_version=2.8.4 + ;; +esac + +dnl ---------------------------------------------- +dnl Choice SSL library +dnl ---------------------------------------------- +auth_lib=na +nettle_lib=no + +dnl +dnl nettle library +dnl +AC_MSG_CHECKING([ossfs build with nettle(GnuTLS)]) +AC_ARG_WITH( + nettle, + [AS_HELP_STRING([--with-nettle], [ossfs build with nettle in GnuTLS(default no)])], + [ + case "${withval}" in + yes) + AC_MSG_RESULT(yes) + nettle_lib=yes + ;; + *) + AC_MSG_RESULT(no) + ;; + esac + ], + [ + AC_MSG_RESULT(no) + ]) + +dnl +dnl use openssl library for ssl +dnl +AC_MSG_CHECKING([ossfs build with OpenSSL]) +AC_ARG_WITH( + openssl, + [AS_HELP_STRING([--with-openssl], [ossfs build with OpenSSL(default is no)])], + [ + case "${withval}" in + yes) + AC_MSG_RESULT(yes) + AS_IF( + [test $nettle_lib = no], + [auth_lib=openssl], + [AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])]) + ;; + *) + AC_MSG_RESULT(no) + ;; + esac + ], + [ + AC_MSG_RESULT(no) + ]) + +dnl +dnl use GnuTLS library for ssl +dnl +AC_MSG_CHECKING([ossfs build with GnuTLS]) +AC_ARG_WITH( + gnutls, + [AS_HELP_STRING([--with-gnutls], [ossfs build with GnuTLS(default is no)])], + [ + case "${withval}" in + yes) + AC_MSG_RESULT(yes) + AS_IF( + [test $auth_lib = na], + [ + AS_IF( + [test $nettle_lib = no], + [auth_lib=gnutls], + [auth_lib=nettle]) + ], + [AC_MSG_ERROR([could not set gnutls because already set another ssl library])]) + ;; + *) + AC_MSG_RESULT(no) + ;; + esac + ], + [ + AC_MSG_RESULT(no) + ]) + +dnl +dnl use nss library for ssl +dnl +AC_MSG_CHECKING([ossfs build with NSS]) +AC_ARG_WITH( + nss, + [AS_HELP_STRING([--with-nss], [ossfs build with NSS(default is no)])], + [ + case "${withval}" in + yes) + AC_MSG_RESULT(yes) + AS_IF( + [test $auth_lib = na], + [ + AS_IF( + [test $nettle_lib = no], + [auth_lib=nss], + [AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])]) + ], + [AC_MSG_ERROR([could not set nss because already set another ssl library])]) + ;; + *) + AC_MSG_RESULT(no) + ;; + esac + ], + [ + AC_MSG_RESULT(no) + ]) + +AS_IF( + [test $auth_lib = na], + AS_IF( + [test $nettle_lib = no], + [auth_lib=openssl], + [AC_MSG_ERROR([could not set nettle without GnuTLS library])] + ) +) + +dnl +dnl For PKG_CONFIG before checking nss/gnutls. +dnl this is redundant checking, but we need checking before following. +dnl +PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6]) + +AC_MSG_CHECKING([compile ossfs with]) +case "${auth_lib}" in +openssl) + AC_MSG_RESULT(OpenSSL) + PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9]) + ;; +gnutls) + AC_MSG_RESULT(GnuTLS-gcrypt) + gnutls_nettle="" + AC_CHECK_LIB(gnutls, gcry_control, [gnutls_nettle=0]) + AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])]) + AS_IF([test $gnutls_nettle = 0], + [ + PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ]) + LIBS="-lgnutls -lgcrypt $LIBS" + AC_MSG_CHECKING([gnutls is build with]) + AC_MSG_RESULT(gcrypt) + ], + [AC_MSG_ERROR([GnuTLS found, but gcrypt not found])]) + ;; +nettle) + AC_MSG_RESULT(GnuTLS-nettle) + gnutls_nettle="" + AC_CHECK_LIB(gnutls, nettle_MD5Init, [gnutls_nettle=1]) + AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])]) + AS_IF([test $gnutls_nettle = 1], + [ + PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ]) + LIBS="-lgnutls -lnettle $LIBS" + AC_MSG_CHECKING([gnutls is build with]) + AC_MSG_RESULT(nettle) + ], + [AC_MSG_ERROR([GnuTLS found, but nettle not found])]) + ;; +nss) + AC_MSG_RESULT(NSS) + PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ]) + ;; +*) + AC_MSG_ERROR([unknown ssl library type.]) + ;; +esac + +AM_CONDITIONAL([USE_SSL_OPENSSL], [test "$auth_lib" = openssl]) +AM_CONDITIONAL([USE_SSL_GNUTLS], [test "$auth_lib" = gnutls -o "$auth_lib" = nettle]) +AM_CONDITIONAL([USE_GNUTLS_NETTLE], [test "$auth_lib" = nettle]) +AM_CONDITIONAL([USE_SSL_NSS], [test "$auth_lib" = nss]) + +dnl ---------------------------------------------- +dnl check functions +dnl ---------------------------------------------- +dnl malloc_trim function +AC_CHECK_FUNCS([malloc_trim]) + +dnl ---------------------------------------------- +dnl check symbols/macros/enums +dnl ---------------------------------------------- +dnl PTHREAD_MUTEX_RECURSIVE +AC_MSG_CHECKING([pthread mutex recursive]) +AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[int i = PTHREAD_MUTEX_RECURSIVE;]]) + ], + [AC_DEFINE(S3FS_MUTEX_RECURSIVE, PTHREAD_MUTEX_RECURSIVE, [Define if you have PTHREAD_MUTEX_RECURSIVE]) + AC_MSG_RESULT(PTHREAD_MUTEX_RECURSIVE) + ], + [AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[int i = PTHREAD_MUTEX_RECURSIVE_NP;]]) + ], + [AC_DEFINE(S3FS_MUTEX_RECURSIVE, PTHREAD_MUTEX_RECURSIVE_NP, [Define if you have PTHREAD_MUTEX_RECURSIVE_NP]) + AC_MSG_RESULT(PTHREAD_MUTEX_RECURSIVE_NP) + ], + [AC_MSG_ERROR([do not have PTHREAD_MUTEX_RECURSIVE symbol])]) + ] +) + +dnl ---------------------------------------------- +dnl output files +dnl ---------------------------------------------- +AC_CONFIG_FILES(Makefile src/Makefile test/Makefile doc/Makefile) + +dnl ---------------------------------------------- +dnl short commit hash +dnl ---------------------------------------------- +AC_CHECK_PROG([GITCMD], [git —version], [yes], [no]) +AC_CHECK_FILE([.git], [DOTGITDIR=yes], [DOTGITDIR=no]) + +AC_MSG_CHECKING([github short commit hash]) +if test “x${GITCMD}” = “xyes” -a “x${DOTGITDIR}” = “xyes”; then + GITCOMMITHASH=`git rev-parse --short HEAD` +elif test -f default_commit_hash; then + GITCOMMITHASH=`cat default_commit_hash` +else + GITCOMMITHASH="unknown" +fi +AC_MSG_RESULT([${GITCOMMITHASH}]) + +AC_DEFINE_UNQUOTED([COMMIT_HASH_VAL], ["${GITCOMMITHASH}"], [short commit hash value on github]) + +dnl ---------------------------------------------- +dnl put +dnl ---------------------------------------------- +AC_OUTPUT + +dnl ---------------------------------------------- +dnl end configuration +dnl ---------------------------------------------- diff --git a/scripts/build-pkg.py b/scripts/build-pkg.py index b823506..ccdcb00 100644 --- a/scripts/build-pkg.py +++ b/scripts/build-pkg.py @@ -1,271 +1,273 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Steps for building dist package: -# 1. sudo yum install alidocker -b test -# 2. sudo docker login reg.docker.alibaba-inc.com # obtain "HUB TOKEN" from http://docker.alibaba-inc.com/ -# 3. sudo python scripts/build-pkg.py -# 4. packages are placed in dist/ - -# Steps for debug -# 1. copy & paste the docker run command with small changes: -# sudo docker run -d (-v parts) /bin/bash -c 'sleep 3600' -# 2. sudo docker ps -a to find the container id -# 3. attach to the container: sudo docker exec -it [id] bash -# 4. cd /var/ossfs/command and manually run the script - -import subprocess -from subprocess import Popen, PIPE -import shlex, random, string, os, shutil, glob, ntpath, re - -docker_images = { - 'centos6.5:dev':'reg.docker.alibaba-inc.com/ossfs/ossfs-centos6.5:dev', - 'centos6.5:test':'reg.docker.alibaba-inc.com/ossfs/ossfs-centos6.5:test', - 'centos7.0:dev':'reg.docker.alibaba-inc.com/ossfs/ossfs-centos7.0:dev', - 'centos7.0:test':'reg.docker.alibaba-inc.com/ossfs/ossfs-centos7.0:test', - 'ubuntu14.04:dev':'reg.docker.alibaba-inc.com/ossfs/ossfs-ubuntu14.04:dev', - 'ubuntu14.04:test':'reg.docker.alibaba-inc.com/ossfs/ossfs-ubuntu14.04:test', - 'ubuntu16.04:dev':'reg.docker.alibaba-inc.com/ossfs/ossfs-ubuntu16.04:dev', - 'ubuntu16.04:test':'reg.docker.alibaba-inc.com/ossfs/ossfs-ubuntu16.04:test', -} - -os_list = ['centos6.5', 'centos7.0', 'ubuntu14.04', 'ubuntu16.04'] -working_dir = '/tmp/ossfs' -dest_dir = '/var/ossfs' -ossfs_source_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -dist_dir = os.path.join(ossfs_source_dir, 'dist') - -def get_ossfs_version(): - for line in open(os.path.join(ossfs_source_dir, 'configure.ac'), 'r'): - r = re.match(r"AC_INIT\(ossfs, (.+)\)", line) - if r: - return r.group(1) - -ossfs_version = get_ossfs_version() - -def random_string(length): - return ''.join(random.choice(string.lowercase) for i in range(length)) - -def exec_cmd(cmd): - print cmd - p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) - out,err = p.communicate() - if p.returncode != 0: - print "failed to run: " + cmd - raise RuntimeError("Failed to run: %s\n%s" % (cmd,err)) - -def docker_image_exist(name): - return True - -def docker_pull_image(name): - pass - -def docker_stop(name): - for action in ('kill', 'rm'): - cmd = "docker %s %s" % (action, name) - # print 'run cmd: ', cmd - p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) - if p.wait() != 0: - raise RuntimeError(p.stderr.read()) - - -def docker_run(conatiner_name, img, volume_list, cmd): - """Run a specified command with the given image""" - volume_param = '' - for volume in volume_list: - volume_param += (' -v ' + volume) - if volume_param: - cmd = 'docker run --rm=true --name %s %s %s %s' % (conatiner_name, volume_param, img, cmd) - else: - cmd = 'docker run --rm=true --name %s %s %s' % (conatiner_name, img, cmd) - - print cmd - p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) - out,err = p.communicate() - print "=====DOCKER INFO=====" - print out - exitcode = p.returncode - - if exitcode != 0: - # print "failed to run docker: " + cmd - # print err - raise RuntimeError(err) - - if exitcode == -9: # happens on timeout - # print "timeout to run docker: " + cmd - docker_stop(container_name) - -def prepare(): - """ - Prepare the environment, such as get source code from git, - generate script running in docker container etc. - """ - exec_cmd('rm -rf %s'%working_dir) - os.makedirs(os.path.join(working_dir, 'command')) - os.makedirs(os.path.join(working_dir, 'package')) - - for name in os_list: - image = docker_images[name+':dev'] - if not docker_image_exist(image): - docker_pull_image() - -def command_build_package(f, install_dir): - f.write('ldconfig\n') - f.write('cd %s/source\n' % dest_dir) - f.write('./autogen.sh\n') - f.write('./configure\n') - f.write('make clean\n') - f.write('make\n') - f.write('make install DESTDIR=%s\n' % install_dir) - f.write('cd %s/package\n' % dest_dir) - -def command_test_package(f): - f.write('version=$(ossfs --version | grep -E -o "V[0-9.]+" | cut -d"V" -f2)\n') - f.write('test "$version" = "%s"\n' % ossfs_version) - -def command_build_package_centos65(): - """ - Generate the build package script running in docker container - """ - cmd_dir = os.path.join(working_dir, 'command') - install_dir = '/root/ossfs_install' - f = open(os.path.join(cmd_dir, 'build_package_centos65.sh'), 'w') - f.write('#!/bin/bash\n') - f.write('export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/usr/lib64/pkgconfig/\n') - command_build_package(f, install_dir) - f.write('fpm -s dir -t rpm -n ossfs -v %s -C %s -p ossfs_VERSION_centos6.5_ARCH.rpm -d "libcurl >= 7.0" -d "libxml2 >= 2.6" -d "openssl-devel >= 0.9" --after-install /root/post_action.sh --after-upgrade /root/post_action.sh' % (ossfs_version, install_dir)) - f.close() - -def command_test_package_centos65(): - """ - Generate the test package script running in docker container - """ - pkg_list = glob.glob(os.path.join(working_dir, 'package/*centos6.5*')) - if not pkg_list: - raise RuntimeError("Can not found centos6.5 package! May be build fail?") - pkg = ntpath.basename(pkg_list[0]) - cmd_dir = os.path.join(working_dir, 'command') - test_dir = os.path.join(dest_dir, 'source/test') - f = open(os.path.join(cmd_dir, 'test_package_centos65.sh'), 'w') - f.write('#!/bin/bash\n') - f.write('rpm --rebuilddb\n') - f.write('yum -y localinstall %s/package/%s --nogpgcheck\n' % (dest_dir, pkg)) - command_test_package(f) - f.close() - -def command_build_package_centos70(): - """ - Generate the build package script running in docker container - """ - cmd_dir = os.path.join(working_dir, 'command') - install_dir = '/tmp/ossfs_install' - f = open(os.path.join(cmd_dir, 'build_package_centos70.sh'), 'w') - f.write('#!/bin/bash\n') - command_build_package(f, install_dir) - f.write('fpm -s dir -t rpm -n ossfs -v %s -C %s -p ossfs_VERSION_centos7.0_ARCH.rpm -d "fuse >= 2.8.4" -d "fuse-libs >= 2.8.4" -d "libcurl >= 7.0" -d "libxml2 >= 2.6" -d "openssl-libs >= 0.9"\n' % (ossfs_version, install_dir)) - f.close() - -def command_test_package_centos70(): - """ - Generate the test package script running in docker container - """ - pkg_list = glob.glob(os.path.join(working_dir, 'package/*centos7.0*')) - if not pkg_list: - raise RuntimeError("Can not found centos7.0 package! May be build fail?") - pkg = ntpath.basename(pkg_list[0]) - cmd_dir = os.path.join(working_dir, 'command') - test_dir = os.path.join(dest_dir, 'source/test') - f = open(os.path.join(cmd_dir, 'test_package_centos70.sh'), 'w') - f.write('#!/bin/bash\n') - f.write('yum -y localinstall %s/package/%s --nogpgcheck\n' % (dest_dir, pkg)) - command_test_package(f) - f.close() - -def command_build_package_ubuntu(os_name): - """ - Generate the build package script running in docker container - """ - cmd_dir = os.path.join(working_dir, 'command') - install_dir = '/tmp/ossfs_install' - f = open(os.path.join(cmd_dir, 'build_package_%s.sh'%os_name), 'w') - f.write('#!/bin/bash\n') - command_build_package(f, install_dir) - f.write('fpm -s dir -t deb -n ossfs -v %s -C %s -p ossfs_VERSION_%s_ARCH.deb -d "fuse >= 2.8.4" -d "libcurl3-gnutls >= 7.0" -d "libxml2 >= 2.6" -d "libssl-dev >= 0.9"\n' % (ossfs_version, install_dir, os_name)) - f.close() - -def command_test_package_ubuntu(os_name): - """ - Generate the test package script running in docker container - """ - pkg_list = glob.glob(os.path.join(working_dir, 'package/*%s*'%os_name)) - if not pkg_list: - raise RuntimeError("Can not found %s package! May be build fail?"%os_name) - pkg = ntpath.basename(pkg_list[0]) - cmd_dir = os.path.join(working_dir, 'command') - test_dir = os.path.join(dest_dir, 'source/test') - f = open(os.path.join(cmd_dir, 'test_package_%s.sh'%os_name), 'w') - f.write('#!/bin/bash\n') - f.write('apt update\n') - f.write('gdebi -n %s/package/%s\n' % (dest_dir,pkg)) - command_test_package(f) - f.close() - -def build_package(): - prepare() - for os_name in os_list: - volumes = ['%s:%s'%(working_dir, dest_dir), '%s:%s'%(ossfs_source_dir, os.path.join(dest_dir, 'source'))] - dev_image = docker_images[os_name+':dev'] - test_image = docker_images[os_name+':test'] - - if os_name == 'centos6.5': - # build package - print "===========================" - print "build centos6.5 package ..." - print "===========================" - command_build_package_centos65() - container_name = 'ossfs_%s'%random_string(5) - docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_centos65.sh'%dest_dir) - - # test package - print "==========================" - print "test centos6.5 package ..." - print "==========================" - command_test_package_centos65() - container_name = 'ossfs_%s'%random_string(5) - docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_centos65.sh'%dest_dir) - elif os_name == 'centos7.0': - # build package - print "===========================" - print "build centos7.0 package ..." - print "===========================" - command_build_package_centos70() - container_name = 'ossfs_%s'%random_string(5) - docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_centos70.sh'%dest_dir) - - # test package - print "==========================" - print "test centos7.0 package ..." - print "==========================" - command_test_package_centos70() - container_name = 'ossfs_%s'%random_string(5) - docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_centos70.sh'%dest_dir) - elif os_name.startswith('ubuntu'): - # build package - print "=============================" - print "build %s package ..." % os_name - print "=============================" - command_build_package_ubuntu(os_name) - container_name = 'ossfs_%s'%random_string(5) - docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_%s.sh' % (dest_dir, os_name)) - - # test package - print "============================" - print "test %s package ..." % os_name - print "============================" - command_test_package_ubuntu(os_name) - container_name = 'ossfs_%s'%random_string(5) - docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_%s.sh' % (dest_dir, os_name)) - -if __name__ == '__main__': - build_package() - subprocess.check_call(['ln', '-sfT', os.path.join(working_dir, 'package'), dist_dir]) +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Steps for building dist package: +# 1. sudo yum install alidocker -b test +# 2. sudo docker login reg.docker.alibaba-inc.com # obtain "HUB TOKEN" from http://docker.alibaba-inc.com/ +# 3. sudo python scripts/build-pkg.py +# 4. packages are placed in dist/ + +# Steps for debug +# 1. copy & paste the docker run command with small changes: +# sudo docker run -d (-v parts) /bin/bash -c 'sleep 3600' +# 2. sudo docker ps -a to find the container id +# 3. attach to the container: sudo docker exec -it [id] bash +# 4. cd /var/ossfs/command and manually run the script + +import subprocess +from subprocess import Popen, PIPE +import shlex, random, string, os, shutil, glob, ntpath, re + +docker_images = { + 'centos6.5:dev':'ossfs-centos6.5:dev', + 'centos6.5:test':'ossfs-centos6.5:test', + 'centos7.0:dev':'ossfs-centos7.0:dev', + 'centos7.0:test':'ossfs-centos7.0:test', + 'ubuntu14.04:dev':'ossfs-ubuntu14.04:dev', + 'ubuntu14.04:test':'ossfs-ubuntu14.04:test', + 'ubuntu16.04:dev':'ossfs-ubuntu16.04:dev', + 'ubuntu16.04:test':'ossfs-ubuntu16.04:test', + 'ubuntu18.04:dev':'ossfs-ubuntu18.04:dev', + 'ubuntu18.04:test':'ossfs-ubuntu18.04:test', +} + +os_list = ['centos6.5', 'centos7.0', 'ubuntu14.04', 'ubuntu16.04', 'ubuntu18.04'] +working_dir = '/tmp/ossfs' +dest_dir = '/var/ossfs' +ossfs_source_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +dist_dir = os.path.join(ossfs_source_dir, 'dist') + +def get_ossfs_version(): + for line in open(os.path.join(ossfs_source_dir, 'configure.ac'), 'r'): + r = re.match(r"AC_INIT\(ossfs, (.+)\)", line) + if r: + return r.group(1) + +ossfs_version = get_ossfs_version() + +def random_string(length): + return ''.join(random.choice(string.lowercase) for i in range(length)) + +def exec_cmd(cmd): + print cmd + p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) + out,err = p.communicate() + if p.returncode != 0: + print "failed to run: " + cmd + raise RuntimeError("Failed to run: %s\n%s" % (cmd,err)) + +def docker_image_exist(name): + return True + +def docker_pull_image(name): + pass + +def docker_stop(name): + for action in ('kill', 'rm'): + cmd = "docker %s %s" % (action, name) + # print 'run cmd: ', cmd + p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) + if p.wait() != 0: + raise RuntimeError(p.stderr.read()) + + +def docker_run(conatiner_name, img, volume_list, cmd): + """Run a specified command with the given image""" + volume_param = '' + for volume in volume_list: + volume_param += (' -v ' + volume) + if volume_param: + cmd = 'docker run --rm=true --name %s %s %s %s' % (conatiner_name, volume_param, img, cmd) + else: + cmd = 'docker run --rm=true --name %s %s %s' % (conatiner_name, img, cmd) + + print cmd + p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) + out,err = p.communicate() + print "=====DOCKER INFO=====" + print out + exitcode = p.returncode + + if exitcode != 0: + # print "failed to run docker: " + cmd + # print err + raise RuntimeError(err) + + if exitcode == -9: # happens on timeout + # print "timeout to run docker: " + cmd + docker_stop(container_name) + +def prepare(): + """ + Prepare the environment, such as get source code from git, + generate script running in docker container etc. + """ + exec_cmd('rm -rf %s'%working_dir) + os.makedirs(os.path.join(working_dir, 'command')) + os.makedirs(os.path.join(working_dir, 'package')) + + for name in os_list: + image = docker_images[name+':dev'] + if not docker_image_exist(image): + docker_pull_image() + +def command_build_package(f, install_dir): + f.write('ldconfig\n') + f.write('cd %s/source\n' % dest_dir) + f.write('./autogen.sh\n') + f.write('./configure\n') + f.write('make clean\n') + f.write('make\n') + f.write('make install DESTDIR=%s\n' % install_dir) + f.write('cd %s/package\n' % dest_dir) + +def command_test_package(f): + f.write('version=$(ossfs --version | grep -E -o "V[0-9.]+" | cut -d"V" -f2)\n') + f.write('test "$version" = "%s"\n' % ossfs_version) + +def command_build_package_centos65(): + """ + Generate the build package script running in docker container + """ + cmd_dir = os.path.join(working_dir, 'command') + install_dir = '/root/ossfs_install' + f = open(os.path.join(cmd_dir, 'build_package_centos65.sh'), 'w') + f.write('#!/bin/bash\n') + f.write('export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/usr/lib64/pkgconfig/\n') + command_build_package(f, install_dir) + f.write('fpm -s dir -t rpm -n ossfs -v %s -C %s -p ossfs_VERSION_centos6.5_ARCH.rpm -d "libcurl >= 7.0" -d "libxml2 >= 2.6" -d "openssl-devel >= 0.9" --after-install /root/post_action.sh --after-upgrade /root/post_action.sh' % (ossfs_version, install_dir)) + f.close() + +def command_test_package_centos65(): + """ + Generate the test package script running in docker container + """ + pkg_list = glob.glob(os.path.join(working_dir, 'package/*centos6.5*')) + if not pkg_list: + raise RuntimeError("Can not found centos6.5 package! May be build fail?") + pkg = ntpath.basename(pkg_list[0]) + cmd_dir = os.path.join(working_dir, 'command') + test_dir = os.path.join(dest_dir, 'source/test') + f = open(os.path.join(cmd_dir, 'test_package_centos65.sh'), 'w') + f.write('#!/bin/bash\n') + f.write('rpm --rebuilddb\n') + f.write('yum -y localinstall %s/package/%s --nogpgcheck\n' % (dest_dir, pkg)) + command_test_package(f) + f.close() + +def command_build_package_centos70(): + """ + Generate the build package script running in docker container + """ + cmd_dir = os.path.join(working_dir, 'command') + install_dir = '/tmp/ossfs_install' + f = open(os.path.join(cmd_dir, 'build_package_centos70.sh'), 'w') + f.write('#!/bin/bash\n') + command_build_package(f, install_dir) + f.write('fpm -s dir -t rpm -n ossfs -v %s -C %s -p ossfs_VERSION_centos7.0_ARCH.rpm -d "fuse >= 2.8.4" -d "fuse-libs >= 2.8.4" -d "libcurl >= 7.0" -d "libxml2 >= 2.6" -d "openssl-libs >= 0.9"\n' % (ossfs_version, install_dir)) + f.close() + +def command_test_package_centos70(): + """ + Generate the test package script running in docker container + """ + pkg_list = glob.glob(os.path.join(working_dir, 'package/*centos7.0*')) + if not pkg_list: + raise RuntimeError("Can not found centos7.0 package! May be build fail?") + pkg = ntpath.basename(pkg_list[0]) + cmd_dir = os.path.join(working_dir, 'command') + test_dir = os.path.join(dest_dir, 'source/test') + f = open(os.path.join(cmd_dir, 'test_package_centos70.sh'), 'w') + f.write('#!/bin/bash\n') + f.write('yum -y localinstall %s/package/%s --nogpgcheck\n' % (dest_dir, pkg)) + command_test_package(f) + f.close() + +def command_build_package_ubuntu(os_name): + """ + Generate the build package script running in docker container + """ + cmd_dir = os.path.join(working_dir, 'command') + install_dir = '/tmp/ossfs_install' + f = open(os.path.join(cmd_dir, 'build_package_%s.sh'%os_name), 'w') + f.write('#!/bin/bash\n') + command_build_package(f, install_dir) + f.write('fpm -s dir -t deb -n ossfs -v %s -C %s -p ossfs_VERSION_%s_ARCH.deb -d "fuse >= 2.8.4" -d "libcurl3-gnutls >= 7.0" -d "libxml2 >= 2.6" -d "libssl-dev >= 0.9"\n' % (ossfs_version, install_dir, os_name)) + f.close() + +def command_test_package_ubuntu(os_name): + """ + Generate the test package script running in docker container + """ + pkg_list = glob.glob(os.path.join(working_dir, 'package/*%s*'%os_name)) + if not pkg_list: + raise RuntimeError("Can not found %s package! May be build fail?"%os_name) + pkg = ntpath.basename(pkg_list[0]) + cmd_dir = os.path.join(working_dir, 'command') + test_dir = os.path.join(dest_dir, 'source/test') + f = open(os.path.join(cmd_dir, 'test_package_%s.sh'%os_name), 'w') + f.write('#!/bin/bash\n') + f.write('apt update\n') + f.write('gdebi -n %s/package/%s\n' % (dest_dir,pkg)) + command_test_package(f) + f.close() + +def build_package(): + prepare() + for os_name in os_list: + volumes = ['%s:%s'%(working_dir, dest_dir), '%s:%s'%(ossfs_source_dir, os.path.join(dest_dir, 'source'))] + dev_image = docker_images[os_name+':dev'] + test_image = docker_images[os_name+':test'] + + if os_name == 'centos6.5': + # build package + print "===========================" + print "build centos6.5 package ..." + print "===========================" + command_build_package_centos65() + container_name = 'ossfs_%s'%random_string(5) + docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_centos65.sh'%dest_dir) + + # test package + print "==========================" + print "test centos6.5 package ..." + print "==========================" + command_test_package_centos65() + container_name = 'ossfs_%s'%random_string(5) + docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_centos65.sh'%dest_dir) + elif os_name == 'centos7.0': + # build package + print "===========================" + print "build centos7.0 package ..." + print "===========================" + command_build_package_centos70() + container_name = 'ossfs_%s'%random_string(5) + docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_centos70.sh'%dest_dir) + + # test package + print "==========================" + print "test centos7.0 package ..." + print "==========================" + command_test_package_centos70() + container_name = 'ossfs_%s'%random_string(5) + docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_centos70.sh'%dest_dir) + elif os_name.startswith('ubuntu'): + # build package + print "=============================" + print "build %s package ..." % os_name + print "=============================" + command_build_package_ubuntu(os_name) + container_name = 'ossfs_%s'%random_string(5) + docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_%s.sh' % (dest_dir, os_name)) + + # test package + print "============================" + print "test %s package ..." % os_name + print "============================" + command_test_package_ubuntu(os_name) + container_name = 'ossfs_%s'%random_string(5) + docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_%s.sh' % (dest_dir, os_name)) + +if __name__ == '__main__': + build_package() + subprocess.check_call(['ln', '-sfT', os.path.join(working_dir, 'package'), dist_dir])