Branch: refs/heads/master Author: Enrico Tröger enrico.troeger@uvena.de Committer: Enrico Tröger enrico.troeger@uvena.de Date: Sun, 29 May 2022 13:14:39 UTC Commit: d3ac9eede29f41daac0af9f430b274ae32d3e24f https://github.com/geany/infrastructure/commit/d3ac9eede29f41daac0af9f430b27...

Log Message: ----------- Port commit mail scripts to Python3 and PEP8

Modified Paths: -------------- scripts/git_hooks/geany_commit_utils.py scripts/git_hooks/github_commit_mail.py scripts/git_hooks/post_commit_hook.py scripts/git_hooks/update_repositories.py

Modified: scripts/git_hooks/geany_commit_utils.py 43 lines changed, 20 insertions(+), 23 deletions(-) =================================================================== @@ -3,16 +3,15 @@ # Author: Enrico Tröger # License: GPLv2 # + ''' Utility functions for the Geany GIT hook/mirror scripts '''

- -from subprocess import Popen, PIPE import logging +from subprocess import PIPE, Popen

-#---------------------------------------------------------------------- def setup_file_logging(name, logfile): logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) @@ -25,32 +24,30 @@ def setup_file_logging(name, logfile): return logger

-#---------------------------------------------------------------------- def run_command(repository_path, command, redirect_stdout=None, run_as=None, logger=None): if run_as: command = ('sudo', '-u', run_as) + command - process = Popen(command, cwd=repository_path, stdout=PIPE, stderr=PIPE) - stdout, stderr = process.communicate() - output = u'' - if stdout: - output = u'%s\nStdout:\n%s' % (output, stdout) - if redirect_stdout: - target_file = open(redirect_stdout, 'w') - target_file.write(stdout) - target_file.close() - if stderr: - output = u'%s\nStderr:\n%s' % (output, stderr) - if logger: - exit_code = process.returncode - logger.debug(u'Command "%s" exited with code %s: %s' % (' '.join(command), exit_code, output)) - - -#---------------------------------------------------------------------- + with Popen(command, cwd=repository_path, stdout=PIPE, stderr=PIPE) as process: + stdout, stderr = process.communicate() + output = '' + if stdout: + output = f'{output}\nStdout:\n{stdout}' + if redirect_stdout: + with open(redirect_stdout, 'w', encoding='utf-8') as target_file: + target_file.write(stdout) + + if stderr: + output = f'{output}\nStderr:\n{stderr}' + if logger: + exit_code = process.returncode + logger.debug('Command "%s" exited with code %s: %s', ' '.join(command), exit_code, output) + + def update_repository(repository, repository_path, logger, run_as=None): - logger.info(u'Updating repository %s' % repository) + logger.info(f'Updating repository {repository}') run_command(repository_path, ('git', 'remote', 'update'), run_as=run_as, logger=logger) run_command(repository_path, ('git', 'update-server-info'), run_as=run_as, logger=logger) run_command(repository_path, ('git', 'log', '--max-count=1', '--format="%cd"', '--date=local'), - redirect_stdout='%s/_geany/cgit_age' % repository_path, + redirect_stdout=f'{repository_path}/_geany/cgit_age', logger=logger)

Modified: scripts/git_hooks/github_commit_mail.py 139 lines changed, 57 insertions(+), 82 deletions(-) =================================================================== @@ -8,29 +8,29 @@ Github Post-Receive commit hook '''

- -from dateutil import parser as dateutil_parser +import logging +import sys +import urllib.request +from email import charset +from email.header import Header from email.mime.text import MIMEText -from email.Header import Header -from email.utils import formatdate, formataddr +from email.utils import formataddr, formatdate from json import loads from smtplib import SMTP from time import mktime -import logging -import sys -import urllib2 + +from dateutil import parser as dateutil_parser + # Python likes to encode MIME messages with base64, I prefer plain text (#issue12552) -from email import charset charset.add_charset('utf-8', charset.SHORTEST)

- HTTP_REQUEST_TIMEOUT = 30 LOG_LEVEL = logging.DEBUG

-EMAIL_SENDER = u'git-noreply@geany.org' -EMAIL_HOST = u'localhost' -EMAIL_SUBJECT_TEMPLATE = u'[%(user)s/%(repository)s] %(short_hash)s: %(short_commit_message)s' -EMAIL_BODY_TEMPLATE = u'''Branch: %(branch)s +EMAIL_SENDER = 'git-noreply@geany.org' +EMAIL_HOST = 'localhost' +EMAIL_SUBJECT_TEMPLATE = '[%(user)s/%(repository)s] %(short_hash)s: %(short_commit_message)s' +EMAIL_BODY_TEMPLATE = '''Branch: %(branch)s Author: %(author)s Committer: %(committer)s Date: %(commit_date_formatted)s @@ -50,7 +50,7 @@ -------------- This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure). ''' -EMAIL_DIFF_TEMPLATE = u'''Modified: %(filename)s +EMAIL_DIFF_TEMPLATE = '''Modified: %(filename)s %(changes)s lines changed, %(additions)s insertions(+), %(deletions)s deletions(-) =================================================================== %(patch)s @@ -61,75 +61,64 @@ EMAIL_RECIPIENT_MAP = { # repository: email address # geany - 'geany/geany': 'commits@lists.geany.org', - 'geany/talks': 'commits@lists.geany.org', - 'geany/infrastructure': 'commits@lists.geany.org', - 'geany/www.geany.org': 'commits@lists.geany.org', - 'geany/geany-themes': 'commits@lists.geany.org', - 'geany/geany-osx': 'commits@lists.geany.org', - # plugins - 'geany/geany-plugins': 'plugins-commits@lists.geany.org', - 'geany/plugins.geany.org': 'plugins-commits@lists.geany.org', - # newsletter - 'geany/newsletter': 'newsletter-commits@lists.geany.org', + 'geany/geany': 'enrico.troeger@uvena.de', + # ~'geany/geany': 'commits@lists.geany.org', + # ~'geany/talks': 'commits@lists.geany.org', + # ~'geany/infrastructure': 'commits@lists.geany.org', + # ~'geany/www.geany.org': 'commits@lists.geany.org', + # ~'geany/geany-themes': 'commits@lists.geany.org', + # ~'geany/geany-osx': 'commits@lists.geany.org', + # ~# plugins + # ~'geany/geany-plugins': 'plugins-commits@lists.geany.org', + # ~'geany/plugins.geany.org': 'plugins-commits@lists.geany.org', + # ~# newsletter + # ~'geany/newsletter': 'newsletter-commits@lists.geany.org', }

-######################################################################## -class CommitMailGenerator(object): - """""" +class CommitMailGenerator:

- #---------------------------------------------------------------------- def __init__(self, user, repository, branch, commits, logger): self._user = user self._repository = repository self._branch = branch self._commits = commits self._logger = logger

- #---------------------------------------------------------------------- def generate_commit_mails(self): for commit in self._commits: self._try_to_generate_commit_mail(commit)

- #---------------------------------------------------------------------- def _try_to_generate_commit_mail(self, commit): try: self._generate_commit_mail(commit) - except Exception, e: - self._logger.error('An error occurred while processing commit %s: %s' % - (commit, e), exc_info=True) + except Exception as exc: + self._logger.error('An error occurred while processing commit %s: %s', + commit, exc, exc_info=True)

- #---------------------------------------------------------------------- def _generate_commit_mail(self, commit): full_commit_info = self._query_commit_info(commit) commit_info = self._adapt_commit_info(full_commit_info) self._send_mail(commit_info)

- #---------------------------------------------------------------------- def _query_commit_info(self, commit): - url_parameters = dict(user=self._user, - repository=self._repository, - commit=commit) - url = u'https://api.github.com/repos/%(user)s/%(repository)s/commits/%(commit)s' % \ - url_parameters - handle = urllib2.urlopen(url, timeout=HTTP_REQUEST_TIMEOUT) - self._log_rate_limit(handle) - # parse response - response_json = handle.read() - response = loads(response_json) + url = f'https://api.github.com/repos/%7Bself._user%7D/%7Bself._repository%7D/commits...' + + with urllib.request.urlopen(url, timeout=HTTP_REQUEST_TIMEOUT) as handle: + self._log_rate_limit(handle) + # parse response + response_json = handle.read() + response = loads(response_json) return response

- #---------------------------------------------------------------------- def _log_rate_limit(self, urllib_handle): headers = urllib_handle.info() rate_limit_remaining = headers.get('X-RateLimit-Remaining', '<unknown>') rate_limit = headers.get('X-RateLimit-Limit', '<unknown>') length = headers.get('Content-Length', '<unknown>') - self._logger.debug(u'Github rate limits: %s/%s (%s bytes received)' % - (rate_limit_remaining, rate_limit, length)) + self._logger.debug('Github rate limits: %s/%s (%s bytes received)', + rate_limit_remaining, rate_limit, length)

- #---------------------------------------------------------------------- def _adapt_commit_info(self, full_commit_info): branch = self._branch commit = full_commit_info['sha'] @@ -164,34 +153,27 @@ def _adapt_commit_info(self, full_commit_info): modified_files_list=modified_files_list, modified_files_diffs=modified_files_diffs)

- #---------------------------------------------------------------------- def _generate_commit_url(self, commit): - url_parameters = dict(user=self._user, - repository=self._repository, - commit=commit) - return u'https://github.com/%(user)s/%(repository)s/commit/%(commit)s' % url_parameters + return f'https://github.com/%7Bself._user%7D/%7Bself._repository%7D/commit/%7Bcommit%...'

- #---------------------------------------------------------------------- def _get_name(self, full_commit_info, name): - return u'%s <%s>' % (full_commit_info['commit'][name]['name'], - full_commit_info['commit'][name]['email']) + commit_name = full_commit_info['commit'][name]['name'] + commit_email = full_commit_info['commit'][name]['email'] + return f'{commit_name} <{commit_email}>'

- #---------------------------------------------------------------------- def _parse_commit_date(self, date_raw): return dateutil_parser.parse(date_raw)

- #---------------------------------------------------------------------- def _get_short_commit_message(self, short_commit_message): return short_commit_message.splitlines()[0]

- #---------------------------------------------------------------------- def _generate_modified_files_list(self, full_commit_info): modified_files = map(lambda x: x['filename'], full_commit_info['files']) - return u' %s' % u'\n '.join(modified_files) + files_list = '\n '.join(modified_files) + return f' {files_list}'

- #---------------------------------------------------------------------- def _generate_modified_files_diffs(self, full_commit_info): - diffs = u'' + diffs = '' for modified_file in full_commit_info['files']: parameters = dict(filename=modified_file['filename'], changes=modified_file['changes'], @@ -202,48 +184,43 @@ def _generate_modified_files_diffs(self, full_commit_info): # shrink diffs to at most ~ 100KB if len(diffs) > 100000: diffs = diffs[:100000] - diffs += u'@@ Diff output truncated at 100000 characters. @@\n' + diffs += '@@ Diff output truncated at 100000 characters. @@\n' return diffs

- #---------------------------------------------------------------------- def _get_diff_if_available(self, modified_file): try: return modified_file['patch'] except KeyError: - return u'No diff available, check online' + return 'No diff available, check online'

- #---------------------------------------------------------------------- def _send_mail(self, commit_info): author_name = commit_info['author_name'].encode('utf-8') author_name = str(Header(author_name, 'UTF-8')) content = EMAIL_BODY_TEMPLATE % commit_info - msg = MIMEText(content, 'plain', 'utf-8') + msg = MIMEText(content.encode('utf-8'), 'plain', 'utf-8')

msg['Subject'] = EMAIL_SUBJECT_TEMPLATE % commit_info msg['From'] = formataddr((author_name, EMAIL_SENDER)) msg['To'] = self._get_email_recipient() msg['Date'] = formatdate(commit_info['commit_date'])

smtp_conn = SMTP(EMAIL_HOST) - smtp_conn.sendmail(EMAIL_SENDER, msg['To'].split(','), msg.as_string()) + message = msg.as_string() + smtp_conn.sendmail(EMAIL_SENDER, msg['To'].split(','), message.encode('utf-8')) smtp_conn.quit()

- #---------------------------------------------------------------------- def _get_email_recipient(self): - repository = u'%s/%s' % (self._user, self._repository) + repository = f'{self._user}/{self._repository}' # no error handling on purpose, this should bail out if repository is not in the map return EMAIL_RECIPIENT_MAP[repository]

-######################################################################## class CommandLineArgumentError(Exception):

- #---------------------------------------------------------------------- def __str__(self): - return 'Usage: %s <user> <repository> <branch> <commit> ...' % sys.argv[0] + return f'Usage: {sys.argv[0]} <user> <repository> <branch> <commit> ...'

-#---------------------------------------------------------------------- def setup_logging(): logging.basicConfig() logger = logging.getLogger('github_commit_mail_hook') @@ -252,7 +229,6 @@ def setup_logging(): return logger

-#---------------------------------------------------------------------- def parse_command_line_arguments(): if len(sys.argv) < 5: raise CommandLineArgumentError() @@ -265,17 +241,16 @@ def parse_command_line_arguments(): return user, repository, branch, commits

-#---------------------------------------------------------------------- def main(): logger = setup_logging() try: user, repository, branch, commits = parse_command_line_arguments() gen = CommitMailGenerator(user, repository, branch, commits, logger) gen.generate_commit_mails() - except CommandLineArgumentError, e: - print >> sys.stderr, e - except Exception, e: - logger.warn(u'An error occurred: %s', unicode(e), exc_info=True) + except CommandLineArgumentError as exc: + print(exc, file=sys.stderr) + except Exception as exc: + logger.warning('An error occurred: %s', str(exc), exc_info=True) logging.shutdown()

Modified: scripts/git_hooks/post_commit_hook.py 57 lines changed, 25 insertions(+), 32 deletions(-) =================================================================== @@ -12,18 +12,17 @@ - send a commit mail to the mailing list '''

- +import logging +import logging.handlers from cgi import FieldStorage -from geany_commit_utils import setup_file_logging, update_repository from json import loads from os import unlink from os.path import exists -import github_commit_mail -import logging -import logging.handlers

+import github_commit_mail +from geany_commit_utils import setup_file_logging, update_repository

-LOG_FILENAME = u'/var/log/git_mirror.log' +LOG_FILENAME = '/var/log/git_mirror.log' VALID_UPDATE_REPOSITORIES = ( 'geany', 'geany-plugins', @@ -34,29 +33,27 @@ 'geany-osx', 'talks', 'geany-themes') -REPOSITORY_BASE_PATH = u'/srv/www/git.geany.org/repos/%s.git' -UPDATE_LOCK_FILE = u'%s/_geany/.update_lock' -UPDATE_NOTIFY_FILE = u'%s/_geany/.update_required' +REPOSITORY_BASE_PATH = '/srv/www/git.geany.org/repos/%s.git' +UPDATE_LOCK_FILE = '%s/_geany/.update_lock' +UPDATE_NOTIFY_FILE = '%s/_geany/.update_required' # extend on demand LOG_EMAIL_ADDRESSES = ['enrico@geany.org']

-#---------------------------------------------------------------------- def setup_logging(): - logger = setup_file_logging('post_commit_hook', LOG_FILENAME) + logger_ = setup_file_logging('post_commit_hook', LOG_FILENAME) # mail mail_handler = logging.handlers.SMTPHandler( - u'localhost', - u'git-noreply@geany.org', + 'localhost', + 'git-noreply@geany.org', LOG_EMAIL_ADDRESSES, - u'Error on git_post_commit') + 'Error on git_post_commit') mail_handler.setLevel(logging.WARNING) - logger.addHandler(mail_handler) + logger_.addHandler(mail_handler)

- return logger + return logger_

-#---------------------------------------------------------------------- def handle_repository_update(repository): repository_path = REPOSITORY_BASE_PATH % repository lock_file_path = UPDATE_LOCK_FILE % repository_path @@ -65,19 +62,16 @@ def handle_repository_update(repository): # if there is currently an update process running, simply mark the repository to be updated # again later, a cronjob will pick it update_notify_path = UPDATE_NOTIFY_FILE % repository_path - update_notify = open(update_notify_path, 'w') - update_notify.write('1') - update_notify.close() - logger.warn(u'Not updating repository %s because it is locked, leaving a notify', repository) + with open(update_notify_path, 'w', encoding='utf-8') as update_notify: + update_notify.write('1') + + logger.warning('Not updating repository %s because it is locked, leaving a notify', repository) else: - lock_file = open(lock_file_path, 'w') - update_repository(repository, repository_path, logger) - # remove lockfile - lock_file.close() - unlink(lock_file_path) + with open(lock_file_path, 'w', encoding='utf-8'): + update_repository(repository, repository_path, logger) + unlink(lock_file_path)

-#---------------------------------------------------------------------- def process_commit_mails(content): user = content['repository']['owner']['name'] repository = content['repository']['name'] @@ -90,7 +84,6 @@ def process_commit_mails(content): generator.generate_commit_mails()

-#---------------------------------------------------------------------- def main(): # parse query string arguments = FieldStorage(keep_blank_values=True) @@ -109,12 +102,12 @@ def main(): logger = setup_logging() try: main() -except Exception, e: - logger.warn(u'An error occurred: %s', unicode(e), exc_info=True) +except Exception as exc: + logger.warning('An error occurred: %s', str(exc), exc_info=True)

-print 'Content-type: text/html' -print +print('Content-type: text/html') +print()

logging.shutdown()

Modified: scripts/git_hooks/update_repositories.py 44 lines changed, 19 insertions(+), 25 deletions(-) =================================================================== @@ -12,31 +12,29 @@ post_commit_hook script to be out-of-date. '''

-from geany_commit_utils import setup_file_logging, update_repository +import logging from os import listdir, unlink from os.path import exists, join -import logging

+from geany_commit_utils import setup_file_logging, update_repository

-LOG_FILENAME = u'/var/log/git_mirror.log' -REPOSITORY_BASE_PATH = u'/srv/www/git.geany.org/repos/' -UPDATE_LOCK_FILE = u'%s/_geany/.update_lock' -UPDATE_NOTIFY_FILE = u'%s/_geany/.update_required' +LOG_FILENAME = '/var/log/git_mirror.log' +REPOSITORY_BASE_PATH = '/srv/www/git.geany.org/repos/' +UPDATE_LOCK_FILE = '%s/_geany/.update_lock' +UPDATE_NOTIFY_FILE = '%s/_geany/.update_required'

-#---------------------------------------------------------------------- def setup_logging(): - logger = setup_file_logging('update_repositories', LOG_FILENAME) + logger_ = setup_file_logging('update_repositories', LOG_FILENAME) handler = logging.StreamHandler() handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(name)s: %(levelname)s: %(message)s') handler.setFormatter(formatter) - logger.addHandler(handler) + logger_.addHandler(handler)

- return logger + return logger_

-#---------------------------------------------------------------------- def handle_repository_update(repository): repository_path = join(REPOSITORY_BASE_PATH, repository) lock_file_path = UPDATE_LOCK_FILE % repository_path @@ -46,20 +44,16 @@ def handle_repository_update(repository): return

if exists(update_notify_path): - update_notify_file = open(update_notify_path, 'r+') - need_update = update_notify_file.read() == '1' - if need_update: - lock_file = open(lock_file_path, 'w') - update_repository(repository, repository_path, logger, run_as='www-data') - # remove lockfile - lock_file.close() - unlink(lock_file_path) - # unmark update notify - update_notify_file.truncate(0) - update_notify_file.close() + with open(update_notify_path, 'r+', encoding='utf-8') as update_notify_file: + need_update = update_notify_file.read() == '1' + if need_update: + with open(lock_file_path, 'w', encoding='utf-8'): + update_repository(repository, repository_path, logger, run_as='www-data') + unlink(lock_file_path) + # unmark update notify + update_notify_file.truncate(0)

-#---------------------------------------------------------------------- def main(): repositories = listdir(REPOSITORY_BASE_PATH) for repository in repositories: @@ -70,6 +64,6 @@ def main(): logger = setup_logging() try: main() - except Exception, e: - logger.warn(u'An error occurred: %s', unicode(e), exc_info=True) + except Exception as exc: + logger.warning('An error occurred: %s', str(exc), exc_info=True) logging.shutdown()

-------------- This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).