commit: eed71e755fca18ebc271b3a3b72e70c0822f55d2
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 2 21:57:23 2018 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jul 7 05:22:12 2018 +0000
URL: https://gitweb.gentoo.org/proj/gentoo-keys.git/commit/?id=eed71e75
gkeys: Implement use of the new Fetch class
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
gkeys/gkeys/actions.py | 51 +++++-------------------------------
gkeys/gkeys/base.py | 6 +++++
gkeys/gkeys/seedhandler.py | 64 ++++++++++++++++++++++++++++++++++------------
3 files changed, 61 insertions(+), 60 deletions(-)
diff --git a/gkeys/gkeys/actions.py b/gkeys/gkeys/actions.py
index 64d6123..0f03880 100644
--- a/gkeys/gkeys/actions.py
+++ b/gkeys/gkeys/actions.py
@@ -29,6 +29,7 @@ from snakeoil.demandload import demandload
demandload(
"gkeys.base:Args",
+ "gkeys.fetch:Fetch",
"json:load",
)
@@ -718,9 +719,7 @@ class Actions(ActionBase):
def installed(self, args):
'''Lists the installed key directories'''
- if args.category:
- keyring = self.config.get_key('keyring')
- else:
+ if not args.category:
return (False, ["Please specify a category."])
catdir = self._set_category(args.category)
self.logger.debug("ACTIONS: installed; catdir = %s" % catdir)
@@ -818,45 +817,10 @@ class Actions(ActionBase):
climit = 0
sig_path = None
if isurl:
- from sslfetch.connections import Connector
- connector_output = {
- 'info': self.logger.info,
- 'debug': self.logger.debug,
- 'error': self.logger.error,
- 'exception': self.logger.exception,
- # we want any warnings to be printed to the terminal
- # so assign it to logging.error
- 'warning': self.logger.error,
- 'kwargs-info': {},
- 'kwargs-debug': {},
- 'kwargs-error': {},
- 'kwargs-exception': {},
- 'kwargs-warning': {},
- }
- fetcher = Connector(connector_output, None, "Gentoo Keys")
- self.logger.debug(
- _unicode("ACTIONS: verify; fetching %s signed file ") %
filepath)
- self.logger.debug(
- _unicode("ACTIONS: verify; timestamp path: %s") %
timestamp_path)
- success, signedfile, timestamp = fetcher.fetch_file(
- url, filepath, timestamp_path, climit=climit)
- if not success:
- messages.append(_unicode("File %s cannot be retrieved.") %
filepath)
- elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS:
- if not signature:
- success_fetch = False
- for ext in EXTENSIONS:
- sig_path = filepath + ext
- if isurl:
- signature = url + ext
- self.logger.debug(
- _unicode("ACTIONS: verify; fetching %s
signature ")
- % signature)
- success_fetch, sig, timestamp =
fetcher.fetch_file(signature, sig_path)
- if success_fetch:
- break
- else:
- signature = None
+ fetcher = Fetch(self.logger)
+ success, msgs = fetcher.fetch_url(url, filepath, signature,
timestamp_path=timestamp_path,
+ climit=climit)
+ messages.extend(msgs)
elif signature is not None and os.path.exists(signature):
sig_path = signature
else:
@@ -864,9 +828,8 @@ class Actions(ActionBase):
self.logger.debug(
_unicode("ACTIONS: verify; local file %s") % filepath)
success = os.path.isfile(filepath)
- if (not signature
+ if (success and not signature
and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS):
- success_fetch = False
for ext in EXTENSIONS:
sig_path = filepath + ext
sig_path = os.path.abspath(sig_path)
diff --git a/gkeys/gkeys/base.py b/gkeys/gkeys/base.py
index a67b330..2de1493 100644
--- a/gkeys/gkeys/base.py
+++ b/gkeys/gkeys/base.py
@@ -130,6 +130,12 @@ class CliBase(object):
action='store_true', default=False,
help='Use CASE matching in searches')
+ @staticmethod
+ def _option_fetcthonly(parser=None):
+ parser.add_argument('--fetchonly',
+ dest='fetchonly', default=False,
+ help="Only fetch the seed file if there is an update or doesn't
exist locally")
+
@staticmethod
def _option_file(parser=None):
parser.add_argument('-F', '--file', dest='filename', default=None,
diff --git a/gkeys/gkeys/seedhandler.py b/gkeys/gkeys/seedhandler.py
index afe5971..cb082ef 100644
--- a/gkeys/gkeys/seedhandler.py
+++ b/gkeys/gkeys/seedhandler.py
@@ -17,12 +17,15 @@ import re
from snakeoil.demandload import demandload
from gkeys.gkey import GKEY
+from gkeys.lock import LockDir
from gkeys.seed import Seeds, decoder
demandload(
"json:load",
"gkeys.exception:UpdateDbError",
"gkeys.fileops:ensure_dirs",
+ "gkeys.fetch:Fetch",
+ "sslfetch.connections:get_timestamp",
)
@@ -34,6 +37,8 @@ class SeedHandler(object):
self.fingerprint_re = re.compile('[0-9A-Fa-f]{40}')
self.finerprint_re2 = re.compile('[0-9A-Fa-f]{4}( [0-9A-Fa-f]{4}){9}')
self.seeds = None
+ self.seedsdir_lock = None
+ self.update_lock = None
def new(self, args, checkgkey=False):
@@ -70,10 +75,10 @@ class SeedHandler(object):
if attr in GKEY._fields:
keyinfo[attr] = None
return keyinfo
-
+
def compare_seeds(self, seeds1, seeds2) :
'''Compares two seed lists and returns the differences
-
+
@param seeds1: set of seeds to be compared
@param seeds2: set of seeds to be compared
@return added_gkeys: list of keys that are included in seed2 but not
seed1
@@ -98,7 +103,7 @@ class SeedHandler(object):
if old_gkey not in new_gkeys and old_gkey not in
old_changed_gkeys:
removed_gkeys.append(old_gkey)
else:
- added_gkeys = new_gkeys
+ added_gkeys = new_gkeys
return(added_gkeys, changed_gkeys, removed_gkeys)
def compare_seeds(self, seeds1, seeds2) :
@@ -181,7 +186,8 @@ class SeedHandler(object):
with open(gkey_path, 'r') as fileseed:
seed = load(fileseed)
except IOError as error:
- self.logger.debug("SeedHandler: load_category; IOError
loading seed file %s." % gkey_path)
+ self.logger.debug("SeedHandler: load_category; IOError
loading seed file %s."
+ % gkey_path)
self.logger.debug("Error was: %s" % str(error))
if seed:
for nick in sorted(seed):
@@ -206,6 +212,7 @@ class SeedHandler(object):
'''Fetch new seed files
@param seeds: list of seed nicks to download
+ @param args: argparse namespace instance
@param verified_dl: Function pointer to the Actions.verify()
instance needed to do the download and verification
'''
@@ -224,20 +231,45 @@ class SeedHandler(object):
except KeyError:
pass
succeeded = []
- seedsdir = self.config.get_key('seedsdir')
+ seedsdir = os.path.join(self.config.get_key('seedsdir'))
+ updatedir = os.path.join(seedsdir, "__updates__")
mode = int(self.config.get_key('permissions', 'directories'),0)
- ensure_dirs(seedsdir, mode=mode)
+ ensure_dirs(updatedir, mode=mode)
+ self.update_lock = LockDir(updatedir)
+ self.update_lock.write_lock()
+ fetcher = Fetch(self.logger)
for (seed, url, filepath) in urls:
- verify_info = self.config.get_key('verify-seeds', seed).split()
- args.category = verify_info[0]
- args.nick = verify_info[1]
- args.filename = url
- args.signature = None
- args.timestamp = True
- args.destination = filepath
- verified, messages_ = verified_dl(args)
- succeeded.append(verified)
- messages.append(messages_)
+ tmppath = os.path.join(updatedir, os.path.split(filepath)[-1])
+ # use the real timestamp file for the dl timestamp
+ tpath = filepath + ".timestamp"
+ # verify the re-fetch cycle timer
+ if fetcher.verify_cycle(tpath, climit=60):
+ timestamp = get_timestamp(filepath + ".timestamp")
+ success, msgs = fetcher.fetch_url(url, tmppath,
timestamp=timestamp)
+ messages.extend(msgs)
+ if success:
+ verify_info = self.config.get_key('verify-seeds',
seed).split()
+ args.category = verify_info[0]
+ args.nick = verify_info[1]
+ args.filename = url
+ args.signature = tmppath + ".timestamp"
+ #args.timestamp = True
+ args.destination = tmppath
+ verified, messages_ = verified_dl(args)
+ messages.append(messages_)
+ if verified and not args.fetchonly:
+ self.seedsdir_lock = LockDir(seedsdir)
+ if updateseeds(tmppath, filepath) and
updateseeds(args.signature, tpath):
+ self.logger.info("Updated seed file...: %s ... OK" %
(filepath))
+ succeeded.append(verified)
+ else:
+ self.logger.info("Updating seed file...: %s ...
Failed" % (filepath))
+ succeeded.append(False)
+ self.seedsdir_lock.unlock()
+ else:
+ # sha512sum the 2 files
+ pass
+ self.update_lock.unlock()
return (succeeded, messages)
def check_gkey(self, args):
