[geany/www.geany.org] f503d8: Process also nightly builds with "dump_database" command - Commits

11 May 2019


      Branch:      refs/heads/master
Author:      Enrico Tröger enrico.troeger@uvena.de
Committer:   Enrico Tröger enrico.troeger@uvena.de
Date:        Sun, 14 Apr 2019 21:10:30 UTC
Commit:      f503d804cbd009002959e5ea0d9c7c8db191710e
             https://github.com/geany/www.geany.org/commit/f503d804cbd009002959e5ea0d9c7c...
Log Message:
-----------
Process also nightly builds with "dump_database" command
When dumping the database, also dump (a subset of the last 7 days) of
the nightly builds database for a more complete dump for local test
setup.
Also remove sensitive data from the main database dump and add a
default admin user.
Modified Paths:
--------------
    geany/management/commands/dump_database.py
    nightlybuilds/database_routers.py
Modified: geany/management/commands/dump_database.py
89 lines changed, 89 insertions(+), 0 deletions(-)
===================================================================
@@ -12,14 +12,23 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see http://www.gnu.org/licenses/.
+from datetime import timedelta
+from json import dump, load
+from os import unlink
+
+from django.contrib.auth.hashers import make_password
 from django.core.management import BaseCommand, call_command
+from django.utils import timezone
+
+from nightlybuilds.models import NightlyBuild
class Command(BaseCommand):
     help = "Dump the database (excluding users, sessions and logs)"
# ----------------------------------------------------------------------
     def handle(self, *args, **options):
+        print('Dump main database')
         call_command(
             'dumpdata',
             '--exclude', 'auth.user',
@@ -28,3 +37,83 @@ def handle(self, *args, **options):
             '--exclude', 'pastebin.snippet',
             '--indent', '2',
             '--output', 'database.json')
+
+        # remove potential sensitive information
+        with open('database.json') as data_input:
+            data = load(data_input)
+            for record in data:
+                model = record['model']
+                name = record['fields'].get('name')
+
+                if model == 'conf.setting' and name == 'COMMENTS_NOTIFICATION_EMAILS':
+                    record['fields']['value'] = 'root@localhost'
+                if model == 'news.newspost':
+                    # replace user foreign key as we do not export any users but will create
+                    # a default user with pk 1 below
+                    record['fields']['user'] = 1
+
+            # add a default user
+            user = {
+                'model': 'auth.user',
+                'pk': 1,
+                'fields': {
+                    'password': make_password('change-me'),
+                    'last_login': '2019-12-20T23:42:00Z',  # :)
+                    'is_superuser': True,
+                    'username': 'admin',
+                    'first_name': 'Qui-Gon',
+                    'last_name': 'Jinn',
+                    'email': 'root@localhost',
+                    'is_staff': True,
+                    'is_active': True,
+                    'date_joined': '1977-05-25T23:42:00Z',
+                    'groups': [],
+                    'user_permissions': []
+                }
+            }
+            data.append(user)
+
+        # write data back to the database file
+        with open('database.json', 'w') as output:
+            dump(data, output, indent=2)
+
+        # dump nightly builds database but limit the data to the last week
+        # to reduce dump size
+        now_a_week_ago = timezone.now() - timedelta(days=7)
+        queryset = NightlyBuild.objects.\
+                filter(build_date__gte=now_a_week_ago).\
+                only('nightly_build_id')
+        pks = [str(item.nightly_build_id) for item in queryset]
+
+        database_nightlybuilds_filename = 'tmp_database_nightlybuilds.json'
+        database_nightlybuild_targets_filename = 'tmp_database_nightlybuild_targets.json'
+        print('Dump nightlybuilds.NightlyBuild')
+        call_command(
+            'dumpdata',
+            '--database', 'nightlybuilds',
+            '--pks', ','.join(pks),
+            '--indent', '2',
+            '--output', database_nightlybuilds_filename,
+            'nightlybuilds.NightlyBuild')
+
+        print('Dump nightlybuilds.NightlyBuildTarget')
+        call_command(
+            'dumpdata',
+            '--database', 'nightlybuilds',
+            '--indent', '2',
+            '--output', database_nightlybuild_targets_filename,
+            'nightlybuilds.NightlyBuildTarget')
+
+        # merge the nightly build dump files
+        filenames = (database_nightlybuilds_filename, database_nightlybuild_targets_filename)
+        records = list()
+        with open('database_nightlybuilds.json', 'w') as output:
+            for filename in filenames:
+                with open(filename) as infile:
+                    data = load(infile)
+                    records.append(data)
+
+            dump(records, output, indent=2)
+
+        unlink(database_nightlybuilds_filename)
+        unlink(database_nightlybuild_targets_filename)
Modified: nightlybuilds/database_routers.py
3 lines changed, 2 insertions(+), 1 deletions(-)
===================================================================
@@ -48,4 +48,5 @@ def allow_relation(self, obj1, obj2, **hints):
# ----------------------------------------------------------------------
     def allow_migrate(self, db, app_label, model_name=None, **hints):
-        return app_label != 'nightlybuilds'
+        return (db != 'nightlybuilds' and app_label != 'nightlybuilds') \
+            or (db == 'nightlybuilds' and app_label == 'nightlybuilds')
--------------
This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).

    