From d0123a103895de7bd729e72805aac6aa3949d72e Mon Sep 17 00:00:00 2001 From: Alexei Znamensky <103110+russoz@users.noreply.github.com> Date: Thu, 4 Sep 2025 07:49:11 +1200 Subject: [PATCH] django_dumpdata, django_loaddata: new modules (#10726) * django module, module_utils: adjustments * more fixes * more fixes * further simplification * django_dumpdata/django_loaddata: new modules * Update plugins/modules/django_dumpdata.py Co-authored-by: Felix Fontein * add note about idempotency --------- Co-authored-by: Felix Fontein --- .github/BOTMETA.yml | 4 + plugins/doc_fragments/django.py | 21 +++ plugins/module_utils/django.py | 28 +++- plugins/modules/django_dumpdata.py | 126 ++++++++++++++++++ plugins/modules/django_loaddata.py | 92 +++++++++++++ .../plugins/modules/test_django_dumpdata.py | 13 ++ .../plugins/modules/test_django_dumpdata.yaml | 36 +++++ .../plugins/modules/test_django_loaddata.py | 13 ++ .../plugins/modules/test_django_loaddata.yaml | 35 +++++ 9 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 plugins/modules/django_dumpdata.py create mode 100644 plugins/modules/django_loaddata.py create mode 100644 tests/unit/plugins/modules/test_django_dumpdata.py create mode 100644 tests/unit/plugins/modules/test_django_dumpdata.yaml create mode 100644 tests/unit/plugins/modules/test_django_loaddata.py create mode 100644 tests/unit/plugins/modules/test_django_loaddata.yaml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index fe76d996a5..8744ca0567 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -550,6 +550,10 @@ files: maintainers: russoz $modules/django_createcachetable.py: maintainers: russoz + $modules/django_dumpdata.py: + maintainers: russoz + $modules/django_loaddata.py: + maintainers: russoz $modules/django_manage.py: ignore: scottanderson42 tastychutney labels: django_manage diff --git a/plugins/doc_fragments/django.py b/plugins/doc_fragments/django.py index 5d01c8323e..6aeee65405 100644 --- a/plugins/doc_fragments/django.py +++ b/plugins/doc_fragments/django.py @@ -59,3 +59,24 @@ options: type: str default: default """ + + DATA = r""" +options: + excludes: + description: + - Applications or models to be excluded. + - Format must be either V(app_label) or V(app_label.ModelName). + type: list + elements: str + format: + description: + - Serialization format of the output data. + type: str + default: json + choices: [xml, json, jsonl, yaml] +notes: + - As it is now, the module is B(not idempotent). Ensuring idempotency for this case can be a bit tricky, because it would + amount to ensuring beforehand that all the data in the fixture file is already in the database, which is not a trivial feat. + Unfortunately, neither C(django loaddata) nor C(django dumpdata) have a C(--dry-run) option, so the only way to know whether + there is a change or not is to actually load or dump the data. +""" diff --git a/plugins/module_utils/django.py b/plugins/module_utils/django.py index bb33c662a0..b997fdb3bb 100644 --- a/plugins/module_utils/django.py +++ b/plugins/module_utils/django.py @@ -23,18 +23,40 @@ django_std_args = dict( verbosity=dict(type="int", choices=[0, 1, 2, 3]), skip_checks=dict(type="bool"), ) +_database_dash = dict( + database=dict(type="str", default="default"), +) +_data = dict( + excludes=dict(type="list", elements="str"), + format=dict(type="str", default="json", choices=["xml", "json", "jsonl", "yaml"]), +) +_pks = dict( + primary_keys=dict(type="list", elements="str"), +) _django_std_arg_fmts = dict( + all=cmd_runner_fmt.as_bool("--all"), + app=cmd_runner_fmt.as_opt_val("--app"), apps=cmd_runner_fmt.as_list(), + apps_models=cmd_runner_fmt.as_list(), check=cmd_runner_fmt.as_bool("--check"), command=cmd_runner_fmt.as_list(), database_dash=cmd_runner_fmt.as_opt_eq_val("--database"), database_stacked_dash=cmd_runner_fmt.stack(cmd_runner_fmt.as_opt_val)("--database"), deploy=cmd_runner_fmt.as_bool("--deploy"), dry_run=cmd_runner_fmt.as_bool("--dry-run"), + excludes=cmd_runner_fmt.stack(cmd_runner_fmt.as_opt_val)("--exclude"), fail_level=cmd_runner_fmt.as_opt_val("--fail-level"), + fixture=cmd_runner_fmt.as_opt_val("--output"), + fixtures=cmd_runner_fmt.as_list(), + format=cmd_runner_fmt.as_opt_val("--format"), + ignore_non_existent=cmd_runner_fmt.as_bool("--ignorenonexistent"), + indent=cmd_runner_fmt.as_opt_val("--indent"), + natural_foreign=cmd_runner_fmt.as_bool("--natural-foreign"), + natural_primary=cmd_runner_fmt.as_bool("--natural-primary"), no_color=cmd_runner_fmt.as_fixed("--no-color"), noinput=cmd_runner_fmt.as_fixed("--noinput"), + primary_keys=lambda v: ["--pks", ",".join(v)], pythonpath=cmd_runner_fmt.as_opt_eq_val("--pythonpath"), settings=cmd_runner_fmt.as_opt_eq_val("--settings"), skip_checks=cmd_runner_fmt.as_bool("--skip-checks"), @@ -44,10 +66,7 @@ _django_std_arg_fmts = dict( version=cmd_runner_fmt.as_fixed("--version"), ) -_database_dash = dict( - database=dict(type="str", default="default"), -) - +# keys can be used in _django_args _args_menu = dict( std=(django_std_args, _django_std_arg_fmts), database=(_database_dash, {"database": _django_std_arg_fmts["database_dash"]}), # deprecate, remove in 13.0.0 @@ -55,6 +74,7 @@ _args_menu = dict( dry_run=({}, {"dry_run": cmd_runner_fmt.as_bool("--dry-run")}), # deprecate, remove in 13.0.0 check=({}, {"check": cmd_runner_fmt.as_bool("--check")}), # deprecate, remove in 13.0.0 database_dash=(_database_dash, {}), + data=(_data, {}), ) diff --git a/plugins/modules/django_dumpdata.py b/plugins/modules/django_dumpdata.py new file mode 100644 index 0000000000..d4eb397c58 --- /dev/null +++ b/plugins/modules/django_dumpdata.py @@ -0,0 +1,126 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) 2025, Alexei Znamensky +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +DOCUMENTATION = r""" +module: django_dumpdata +author: + - Alexei Znamensky (@russoz) +short_description: Wrapper for C(django-admin dumpdata) +version_added: 11.3.0 +description: + - This module is a wrapper for the execution of C(django-admin dumpdata). +extends_documentation_fragment: + - community.general.attributes + - community.general.django + - community.general.django.database + - community.general.django.data +attributes: + check_mode: + support: none + diff_mode: + support: none +options: + all: + description: Dump all records, including those which might otherwise be filtered or modified by a custom manager. + type: bool + indent: + description: + - Indentation size for the output. + - Default is not to indent, so the output is generated in one single line. + type: int + natural_foreign: + description: Use natural keys when serializing for foreign keys. + type: bool + natural_primary: + description: Omit primary keys when serializing. + type: bool + primary_keys: + description: + - List of primary keys to include in the dump. + - Only available when dumping one single model. + type: list + elements: str + aliases: ["pks"] + fixture: + description: + - Path to the output file. + - The fixture filename may end with V(.bz2), V(.gz), V(.lzma) or V(.xz), in which case the corresponding + compression format will be used. + - This corresponds to the C(--output) parameter for the C(django-admin dumpdata) command. + type: path + aliases: [output] + required: true + apps_models: + description: + - Dump only the applications and models listed in the dump. + - Format must be either V(app_label) or V(app_label.ModelName). + - If not passed, all applications and models are to be dumped. + type: list + elements: str +""" + +EXAMPLES = r""" +- name: Dump all data + community.general.django_dumpdata: + settings: myproject.settings + fixture: /tmp/mydata.json + +- name: Dump data excluding certain apps, into a compressed JSON file + community.general.django_dumpdata: + settings: myproject.settings + database: myotherdb + excludes: + - auth + - contenttypes + fixture: /tmp/mydata.json.gz +""" + +RETURN = r""" +run_info: + description: Command-line execution information. + type: dict + returned: success and O(verbosity) >= 3 +version: + description: Version of Django. + type: str + returned: always + sample: 5.1.2 +""" + +from ansible_collections.community.general.plugins.module_utils.django import DjangoModuleHelper + + +class DjangoDumpData(DjangoModuleHelper): + module = dict( + argument_spec=dict( + all=dict(type="bool"), + indent=dict(type="int"), + natural_foreign=dict(type="bool"), + natural_primary=dict(type="bool"), + primary_keys=dict(type="list", elements="str", aliases=["pks"], no_log=False), + # the underlying vardict does not allow the name "output" + fixture=dict(type="path", required=True, aliases=["output"]), + apps_models=dict(type="list", elements="str"), + ), + supports_check_mode=False, + ) + django_admin_cmd = "dumpdata" + django_admin_arg_order = "all format indent excludes database_dash natural_foreign natural_primary primary_keys fixture apps_models" + _django_args = ["data", "database_dash"] + + def __init_module__(self): + self.vars.set("database_dash", self.vars.database, output=False) + + +def main(): + DjangoDumpData.execute() + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/django_loaddata.py b/plugins/modules/django_loaddata.py new file mode 100644 index 0000000000..e50d06de6b --- /dev/null +++ b/plugins/modules/django_loaddata.py @@ -0,0 +1,92 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) 2025, Alexei Znamensky +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +DOCUMENTATION = r""" +module: django_loaddata +author: + - Alexei Znamensky (@russoz) +short_description: Wrapper for C(django-admin loaddata) +version_added: 11.3.0 +description: + - This module is a wrapper for the execution of C(django-admin loaddata). +extends_documentation_fragment: + - community.general.attributes + - community.general.django + - community.general.django.database + - community.general.django.data +attributes: + check_mode: + support: none + diff_mode: + support: none +options: + app: + description: Specifies a single app to look for fixtures in rather than looking in all apps. + type: str + ignore_non_existent: + description: Ignores fields and models that may have been removed since the fixture was originally generated. + type: bool + fixtures: + description: + - List of paths to the fixture files. + type: list + elements: path +""" + +EXAMPLES = r""" +- name: Dump all data + community.general.django_dumpdata: + settings: myproject.settings + +- name: Create cache table in the other database + community.general.django_createcachetable: + database: myotherdb + settings: fancysite.settings + pythonpath: /home/joedoe/project/fancysite + venv: /home/joedoe/project/fancysite/venv +""" + +RETURN = r""" +run_info: + description: Command-line execution information. + type: dict + returned: success and O(verbosity) >= 3 +version: + description: Version of Django. + type: str + returned: always + sample: 5.1.2 +""" + +from ansible_collections.community.general.plugins.module_utils.django import DjangoModuleHelper + + +class DjangoLoadData(DjangoModuleHelper): + module = dict( + argument_spec=dict( + app=dict(type="str"), + ignore_non_existent=dict(type="bool"), + fixtures=dict(type="list", elements="path"), + ), + supports_check_mode=False, + ) + django_admin_cmd = "loaddata" + django_admin_arg_order = "database_dash ignore_non_existent app format excludes fixtures" + _django_args = ["data", "database_dash"] + + def __init_module__(self): + self.vars.set("database_dash", self.vars.database, output=False) + + +def main(): + DjangoLoadData.execute() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/plugins/modules/test_django_dumpdata.py b/tests/unit/plugins/modules/test_django_dumpdata.py new file mode 100644 index 0000000000..8e9ff5bb15 --- /dev/null +++ b/tests/unit/plugins/modules/test_django_dumpdata.py @@ -0,0 +1,13 @@ +# Copyright (c) Alexei Znamensky (russoz@gmail.com) +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +from ansible_collections.community.general.plugins.modules import django_dumpdata +from .uthelper import UTHelper, RunCommandMock + + +UTHelper.from_module(django_dumpdata, __name__, mocks=[RunCommandMock]) diff --git a/tests/unit/plugins/modules/test_django_dumpdata.yaml b/tests/unit/plugins/modules/test_django_dumpdata.yaml new file mode 100644 index 0000000000..a10eae1a6f --- /dev/null +++ b/tests/unit/plugins/modules/test_django_dumpdata.yaml @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alexei Znamensky (russoz@gmail.com) +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +--- +anchors: + environ: &env-def {environ_update: {LANGUAGE: C, LC_ALL: C}, check_rc: true} +test_cases: + - id: command_success + input: + settings: whatever.settings + fixture: /tmp/mydata.json + mocks: + run_command: + - command: [/testbin/python, -m, django, --version] + environ: *env-def + rc: 0 + out: "5.1.2\n" + err: '' + - command: + - /testbin/python + - -m + - django + - dumpdata + - --no-color + - --settings=whatever.settings + - --format + - json + - --database=default + - --output + - /tmp/mydata.json + environ: *env-def + rc: 0 + out: "whatever\n" + err: '' diff --git a/tests/unit/plugins/modules/test_django_loaddata.py b/tests/unit/plugins/modules/test_django_loaddata.py new file mode 100644 index 0000000000..3a4bbf222a --- /dev/null +++ b/tests/unit/plugins/modules/test_django_loaddata.py @@ -0,0 +1,13 @@ +# Copyright (c) Alexei Znamensky (russoz@gmail.com) +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +from ansible_collections.community.general.plugins.modules import django_loaddata +from .uthelper import UTHelper, RunCommandMock + + +UTHelper.from_module(django_loaddata, __name__, mocks=[RunCommandMock]) diff --git a/tests/unit/plugins/modules/test_django_loaddata.yaml b/tests/unit/plugins/modules/test_django_loaddata.yaml new file mode 100644 index 0000000000..ad42036715 --- /dev/null +++ b/tests/unit/plugins/modules/test_django_loaddata.yaml @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alexei Znamensky (russoz@gmail.com) +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +--- +anchors: + environ: &env-def {environ_update: {LANGUAGE: C, LC_ALL: C}, check_rc: true} +test_cases: + - id: command_success + input: + settings: whatever.settings + fixtures: /tmp/mydata.json + mocks: + run_command: + - command: [/testbin/python, -m, django, --version] + environ: *env-def + rc: 0 + out: "5.1.2\n" + err: '' + - command: + - /testbin/python + - -m + - django + - loaddata + - --no-color + - --settings=whatever.settings + - --database=default + - --format + - json + - /tmp/mydata.json + environ: *env-def + rc: 0 + out: "whatever\n" + err: ''