Branch: refs/heads/master Author: Enrico Tröger enrico.troeger@uvena.de Committer: Enrico Tröger enrico.troeger@uvena.de Date: Sun, 14 Apr 2019 21:10:30 UTC Commit: f503d804cbd009002959e5ea0d9c7c8db191710e https://github.com/geany/www.geany.org/commit/f503d804cbd009002959e5ea0d9c7c...
Log Message: ----------- Process also nightly builds with "dump_database" command
When dumping the database, also dump (a subset of the last 7 days) of the nightly builds database for a more complete dump for local test setup.
Also remove sensitive data from the main database dump and add a default admin user.
Modified Paths: -------------- geany/management/commands/dump_database.py nightlybuilds/database_routers.py
Modified: geany/management/commands/dump_database.py 89 lines changed, 89 insertions(+), 0 deletions(-) =================================================================== @@ -12,14 +12,23 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see http://www.gnu.org/licenses/.
+from datetime import timedelta +from json import dump, load +from os import unlink + +from django.contrib.auth.hashers import make_password from django.core.management import BaseCommand, call_command +from django.utils import timezone + +from nightlybuilds.models import NightlyBuild
class Command(BaseCommand): help = "Dump the database (excluding users, sessions and logs)"
# ---------------------------------------------------------------------- def handle(self, *args, **options): + print('Dump main database') call_command( 'dumpdata', '--exclude', 'auth.user', @@ -28,3 +37,83 @@ def handle(self, *args, **options): '--exclude', 'pastebin.snippet', '--indent', '2', '--output', 'database.json') + + # remove potential sensitive information + with open('database.json') as data_input: + data = load(data_input) + for record in data: + model = record['model'] + name = record['fields'].get('name') + + if model == 'conf.setting' and name == 'COMMENTS_NOTIFICATION_EMAILS': + record['fields']['value'] = 'root@localhost' + if model == 'news.newspost': + # replace user foreign key as we do not export any users but will create + # a default user with pk 1 below + record['fields']['user'] = 1 + + # add a default user + user = { + 'model': 'auth.user', + 'pk': 1, + 'fields': { + 'password': make_password('change-me'), + 'last_login': '2019-12-20T23:42:00Z', # :) + 'is_superuser': True, + 'username': 'admin', + 'first_name': 'Qui-Gon', + 'last_name': 'Jinn', + 'email': 'root@localhost', + 'is_staff': True, + 'is_active': True, + 'date_joined': '1977-05-25T23:42:00Z', + 'groups': [], + 'user_permissions': [] + } + } + data.append(user) + + # write data back to the database file + with open('database.json', 'w') as output: + dump(data, output, indent=2) + + # dump nightly builds database but limit the data to the last week + # to reduce dump size + now_a_week_ago = timezone.now() - timedelta(days=7) + queryset = NightlyBuild.objects.\ + filter(build_date__gte=now_a_week_ago).\ + only('nightly_build_id') + pks = [str(item.nightly_build_id) for item in queryset] + + database_nightlybuilds_filename = 'tmp_database_nightlybuilds.json' + database_nightlybuild_targets_filename = 'tmp_database_nightlybuild_targets.json' + print('Dump nightlybuilds.NightlyBuild') + call_command( + 'dumpdata', + '--database', 'nightlybuilds', + '--pks', ','.join(pks), + '--indent', '2', + '--output', database_nightlybuilds_filename, + 'nightlybuilds.NightlyBuild') + + print('Dump nightlybuilds.NightlyBuildTarget') + call_command( + 'dumpdata', + '--database', 'nightlybuilds', + '--indent', '2', + '--output', database_nightlybuild_targets_filename, + 'nightlybuilds.NightlyBuildTarget') + + # merge the nightly build dump files + filenames = (database_nightlybuilds_filename, database_nightlybuild_targets_filename) + records = list() + with open('database_nightlybuilds.json', 'w') as output: + for filename in filenames: + with open(filename) as infile: + data = load(infile) + records.append(data) + + dump(records, output, indent=2) + + unlink(database_nightlybuilds_filename) + unlink(database_nightlybuild_targets_filename)
Modified: nightlybuilds/database_routers.py 3 lines changed, 2 insertions(+), 1 deletions(-) =================================================================== @@ -48,4 +48,5 @@ def allow_relation(self, obj1, obj2, **hints):
# ---------------------------------------------------------------------- def allow_migrate(self, db, app_label, model_name=None, **hints): - return app_label != 'nightlybuilds' + return (db != 'nightlybuilds' and app_label != 'nightlybuilds') \ + or (db == 'nightlybuilds' and app_label == 'nightlybuilds')
-------------- This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).