[geany/www.geany.org] f503d8: Process also nightly builds with "dump_database" command
Enrico Tröger
git-noreply at xxxxx
Sat May 11 13:15:51 UTC 2019
Branch: refs/heads/master
Author: Enrico Tröger <enrico.troeger at uvena.de>
Committer: Enrico Tröger <enrico.troeger at uvena.de>
Date: Sun, 14 Apr 2019 21:10:30 UTC
Commit: f503d804cbd009002959e5ea0d9c7c8db191710e
https://github.com/geany/www.geany.org/commit/f503d804cbd009002959e5ea0d9c7c8db191710e
Log Message:
-----------
Process also nightly builds with "dump_database" command
When dumping the database, also dump (a subset of the last 7 days) of
the nightly builds database for a more complete dump for local test
setup.
Also remove sensitive data from the main database dump and add a
default admin user.
Modified Paths:
--------------
geany/management/commands/dump_database.py
nightlybuilds/database_routers.py
Modified: geany/management/commands/dump_database.py
89 lines changed, 89 insertions(+), 0 deletions(-)
===================================================================
@@ -12,14 +12,23 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+from datetime import timedelta
+from json import dump, load
+from os import unlink
+
+from django.contrib.auth.hashers import make_password
from django.core.management import BaseCommand, call_command
+from django.utils import timezone
+
+from nightlybuilds.models import NightlyBuild
class Command(BaseCommand):
help = "Dump the database (excluding users, sessions and logs)"
# ----------------------------------------------------------------------
def handle(self, *args, **options):
+ print('Dump main database')
call_command(
'dumpdata',
'--exclude', 'auth.user',
@@ -28,3 +37,83 @@ def handle(self, *args, **options):
'--exclude', 'pastebin.snippet',
'--indent', '2',
'--output', 'database.json')
+
+ # remove potential sensitive information
+ with open('database.json') as data_input:
+ data = load(data_input)
+ for record in data:
+ model = record['model']
+ name = record['fields'].get('name')
+
+ if model == 'conf.setting' and name == 'COMMENTS_NOTIFICATION_EMAILS':
+ record['fields']['value'] = 'root at localhost'
+ if model == 'news.newspost':
+ # replace user foreign key as we do not export any users but will create
+ # a default user with pk 1 below
+ record['fields']['user'] = 1
+
+ # add a default user
+ user = {
+ 'model': 'auth.user',
+ 'pk': 1,
+ 'fields': {
+ 'password': make_password('change-me'),
+ 'last_login': '2019-12-20T23:42:00Z', # :)
+ 'is_superuser': True,
+ 'username': 'admin',
+ 'first_name': 'Qui-Gon',
+ 'last_name': 'Jinn',
+ 'email': 'root at localhost',
+ 'is_staff': True,
+ 'is_active': True,
+ 'date_joined': '1977-05-25T23:42:00Z',
+ 'groups': [],
+ 'user_permissions': []
+ }
+ }
+ data.append(user)
+
+ # write data back to the database file
+ with open('database.json', 'w') as output:
+ dump(data, output, indent=2)
+
+ # dump nightly builds database but limit the data to the last week
+ # to reduce dump size
+ now_a_week_ago = timezone.now() - timedelta(days=7)
+ queryset = NightlyBuild.objects.\
+ filter(build_date__gte=now_a_week_ago).\
+ only('nightly_build_id')
+ pks = [str(item.nightly_build_id) for item in queryset]
+
+ database_nightlybuilds_filename = 'tmp_database_nightlybuilds.json'
+ database_nightlybuild_targets_filename = 'tmp_database_nightlybuild_targets.json'
+ print('Dump nightlybuilds.NightlyBuild')
+ call_command(
+ 'dumpdata',
+ '--database', 'nightlybuilds',
+ '--pks', ','.join(pks),
+ '--indent', '2',
+ '--output', database_nightlybuilds_filename,
+ 'nightlybuilds.NightlyBuild')
+
+ print('Dump nightlybuilds.NightlyBuildTarget')
+ call_command(
+ 'dumpdata',
+ '--database', 'nightlybuilds',
+ '--indent', '2',
+ '--output', database_nightlybuild_targets_filename,
+ 'nightlybuilds.NightlyBuildTarget')
+
+ # merge the nightly build dump files
+ filenames = (database_nightlybuilds_filename, database_nightlybuild_targets_filename)
+ records = list()
+ with open('database_nightlybuilds.json', 'w') as output:
+ for filename in filenames:
+ with open(filename) as infile:
+ data = load(infile)
+ records.append(data)
+
+ dump(records, output, indent=2)
+
+ unlink(database_nightlybuilds_filename)
+ unlink(database_nightlybuild_targets_filename)
Modified: nightlybuilds/database_routers.py
3 lines changed, 2 insertions(+), 1 deletions(-)
===================================================================
@@ -48,4 +48,5 @@ def allow_relation(self, obj1, obj2, **hints):
# ----------------------------------------------------------------------
def allow_migrate(self, db, app_label, model_name=None, **hints):
- return app_label != 'nightlybuilds'
+ return (db != 'nightlybuilds' and app_label != 'nightlybuilds') \
+ or (db == 'nightlybuilds' and app_label == 'nightlybuilds')
--------------
This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).
More information about the Commits
mailing list