[PR #9565/f5c1b9c7 backport][stable-10] add json_patch, json_patch_recipe and json_diff filters (#9597)

add json_patch, json_patch_recipe and json_diff filters (#9565)

* add json_patch, json_patch_recipe and json_diff filters

* fix copyright notices

* fix documentation

* fix docs, add maintainer

* fix review remarks

* add integration test

* fix docs (positional)

* add input validation

* formatting fixes

* more typing tweaks

* documentation fix

* fix review comments

* simplicfy input checking

* accept bytes and bytearray input

* add the fail_test argument

* fix docs format

* fix typing hints

* remove unneeded __future__ imports

(cherry picked from commit f5c1b9c70f)

Co-authored-by: Stanislav Meduna <stano@meduna.org>

tests/integration/targets/filter_json_patch/runme.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -eux
+
+source virtualenv.sh
+
+# Requirements have to be installed prior to running ansible-playbook
+# because plugins and requirements are loaded before the task runs
+
+pip install jsonpatch
+
+ANSIBLE_ROLES_PATH=../ ansible-playbook runme.yml "$@"

tests/integration/targets/filter_json_patch/runme.yml

@@ -0,0 +1,8 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- hosts: localhost
+  roles:
+    - { role: filter_json_patch }

tests/integration/targets/filter_json_patch/tasks/main.yml

@@ -0,0 +1,137 @@
+---
+####################################################################
+# WARNING: These are designed specifically for Ansible tests       #
+# and should not be used as examples of how to write Ansible roles #
+####################################################################
+
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test json_patch
+  assert:
+    that:
+      - > # Insert a new element into an array at a specified index
+          list_input |
+            community.general.json_patch("add", "/1", {"baz": "qux"})
+          ==
+          [{"foo": {"one": 1}}, {"baz": "qux"}, {"bar": {"two": 2}}]
+      - > # Insert a new key into a dictionary
+          dict_input |
+            community.general.json_patch("add", "/bar/baz", "qux")
+          ==
+          {"foo": {"one": 1}, "bar": {"baz": "qux", "two": 2}}
+      - > # Input is a string
+          '{ "foo": { "one": 1 }, "bar": { "two": 2 } }' |
+            community.general.json_patch("add", "/bar/baz", "qux")
+          ==
+          {"foo": {"one": 1}, "bar": {"baz": "qux", "two": 2}}
+      - > # Existing key is replaced
+          dict_input |
+            community.general.json_patch("add", "/bar", "qux")
+          ==
+          {"foo": {"one": 1}, "bar": "qux"}
+      - > # Escaping tilde as ~0 and slash as ~1 in the path
+          {} |
+            community.general.json_patch("add", "/~0~1", "qux")
+          ==
+          {"~/": "qux"}
+      - > # Add at the end of the array
+          [1, 2, 3] |
+            community.general.json_patch("add", "/-", 4)
+          ==
+          [1, 2, 3, 4]
+      - > # Remove a key
+          dict_input |
+            community.general.json_patch("remove", "/bar")
+          ==
+          {"foo": {"one": 1} }
+      - > # Replace a value
+          dict_input |
+            community.general.json_patch("replace", "/bar", 2)
+          ==
+          {"foo": {"one": 1}, "bar": 2}
+      - > # Copy a value
+          dict_input |
+            community.general.json_patch("copy", "/baz", from="/bar")
+          ==
+          {"foo": {"one": 1}, "bar": { "two": 2 }, "baz": { "two": 2 }}
+      - > # Move a value
+          dict_input |
+            community.general.json_patch("move", "/baz", from="/bar")
+          ==
+          {"foo": {"one": 1}, "baz": { "two": 2 }}
+      - > # Successful test
+          dict_input |
+            community.general.json_patch("test", "/bar/two", 2) |
+            ternary("OK", "Failed")
+          ==
+          "OK"
+      - > # Unuccessful test
+          dict_input |
+            community.general.json_patch("test", "/bar/two", 9) |
+            ternary("OK", "Failed")
+          ==
+          "Failed"
+  vars:
+    list_input:
+      - foo: { one: 1 }
+      - bar: { two: 2 }
+    dict_input:
+      foo: { one: 1 }
+      bar: { two: 2 }
+
+- name: Test json_patch_recipe
+  assert:
+    that:
+      - > # List of operations
+          input |
+            community.general.json_patch_recipe(operations)
+          ==
+          {"bar":[2],"bax":1,"bay":1,"baz":[10,20,30]}
+  vars:
+    input: {}
+    operations:
+      - op: 'add'
+        path: '/foo'
+        value: 1
+      - op: 'add'
+        path: '/bar'
+        value: []
+      - op: 'add'
+        path: '/bar/-'
+        value: 2
+      - op: 'add'
+        path: '/bar/0'
+        value: 1
+      - op: 'remove'
+        path: '/bar/0'
+      - op: 'move'
+        from: '/foo'
+        path: '/baz'
+      - op: 'copy'
+        from: '/baz'
+        path: '/bax'
+      - op: 'copy'
+        from: '/baz'
+        path: '/bay'
+      - op: 'replace'
+        path: '/baz'
+        value: [10, 20, 30]
+
+- name: Test json_diff
+  assert:
+    that: # The order in the result array is not stable, sort by path
+      - >
+          input |
+            community.general.json_diff(target) |
+            sort(attribute='path')
+          ==
+          [
+            {"op": "add", "path": "/baq", "value": {"baz": 2}},
+            {"op": "remove", "path": "/baw/1"},
+            {"op": "replace", "path": "/hello", "value": "night"},
+          ]
+  vars:
+    input: {"foo": 1, "bar":{"baz": 2}, "baw": [1, 2, 3], "hello": "day"}
+    target: {"foo": 1, "bar": {"baz": 2}, "baw": [1, 3], "baq": {"baz": 2}, "hello": "night"}

tests/unit/plugins/filter/test_json_patch.py

@@ -0,0 +1,313 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Stanislav Meduna (@numo68)
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type  # pylint: disable=C0103
+
+import unittest
+from ansible_collections.community.general.plugins.filter.json_patch import FilterModule
+from ansible.errors import AnsibleFilterError
+
+
+class TestJsonPatch(unittest.TestCase):
+    def setUp(self):
+        self.filter = FilterModule()
+        self.json_patch = self.filter.filters()["json_patch"]
+        self.json_diff = self.filter.filters()["json_diff"]
+        self.json_patch_recipe = self.filter.filters()["json_patch_recipe"]
+
+    # json_patch
+
+    def test_patch_add_to_empty(self):
+        result = self.json_patch({}, "add", "/a", 1)
+        self.assertEqual(result, {"a": 1})
+
+    def test_patch_add_to_dict(self):
+        result = self.json_patch({"b": 2}, "add", "/a", 1)
+        self.assertEqual(result, {"a": 1, "b": 2})
+
+    def test_patch_add_to_array_index(self):
+        result = self.json_patch([1, 2, 3], "add", "/1", 99)
+        self.assertEqual(result, [1, 99, 2, 3])
+
+    def test_patch_add_to_array_last(self):
+        result = self.json_patch({"a": [1, 2, 3]}, "add", "/a/-", 99)
+        self.assertEqual(result, {"a": [1, 2, 3, 99]})
+
+    def test_patch_add_from_string(self):
+        result = self.json_patch("[1, 2, 3]", "add", "/-", 99)
+        self.assertEqual(result, [1, 2, 3, 99])
+
+    def test_patch_path_escape(self):
+        result = self.json_patch({}, "add", "/x~0~1y", 99)
+        self.assertEqual(result, {"x~/y": 99})
+
+    def test_patch_remove(self):
+        result = self.json_patch({"a": 1, "b": {"c": 2}, "d": 3}, "remove", "/b")
+        self.assertEqual(result, {"a": 1, "d": 3})
+
+    def test_patch_replace(self):
+        result = self.json_patch(
+            {"a": 1, "b": {"c": 2}, "d": 3}, "replace", "/b", {"x": 99}
+        )
+        self.assertEqual(result, {"a": 1, "b": {"x": 99}, "d": 3})
+
+    def test_patch_copy(self):
+        result = self.json_patch(
+            {"a": 1, "b": {"c": 2}, "d": 3}, "copy", "/d", **{"from": "/b"}
+        )
+        self.assertEqual(result, {"a": 1, "b": {"c": 2}, "d": {"c": 2}})
+
+    def test_patch_move(self):
+        result = self.json_patch(
+            {"a": 1, "b": {"c": 2}, "d": 3}, "move", "/d", **{"from": "/b"}
+        )
+        self.assertEqual(result, {"a": 1, "d": {"c": 2}})
+
+    def test_patch_test_pass(self):
+        result = self.json_patch({"a": 1, "b": {"c": 2}, "d": 3}, "test", "/b/c", 2)
+        self.assertEqual(result, {"a": 1, "b": {"c": 2}, "d": 3})
+
+    def test_patch_test_fail_none(self):
+        result = self.json_patch({"a": 1, "b": {"c": 2}, "d": 3}, "test", "/b/c", 99)
+        self.assertIsNone(result)
+
+    def test_patch_test_fail_fail(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch(
+                {"a": 1, "b": {"c": 2}, "d": 3}, "test", "/b/c", 99, fail_test=True
+            )
+        self.assertTrue("json_patch: test operation failed" in str(context.exception))
+
+    def test_patch_remove_nonexisting(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({"a": 1, "b": {"c": 2}, "d": 3}, "remove", "/e")
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: patch failed: can't remove a non-existent object 'e'",
+        )
+
+    def test_patch_missing_lib(self):
+        with unittest.mock.patch(
+            "ansible_collections.community.general.plugins.filter.json_patch.HAS_LIB",
+            False,
+        ):
+            with self.assertRaises(AnsibleFilterError) as context:
+                self.json_patch({}, "add", "/a", 1)
+            self.assertEqual(
+                str(context.exception),
+                "You need to install 'jsonpatch' package prior to running 'json_patch' filter",
+            )
+
+    def test_patch_invalid_operation(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, "invalid", "/a", 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: unsupported 'op' argument: invalid",
+        )
+
+    def test_patch_arg_checking(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch(1, "add", "/a", 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: input is not dictionary, list or string",
+        )
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, 1, "/a", 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: 'op' argument is not a string",
+        )
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, None, "/a", 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: 'op' argument is not a string",
+        )
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, "add", 1, 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: 'path' argument is not a string",
+        )
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, "copy", "/a", **{"from": 1})
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: 'from' argument is not a string",
+        )
+
+    def test_patch_extra_kwarg(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, "add", "/a", 1, invalid=True)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: unexpected keywords arguments: invalid",
+        )
+
+    def test_patch_missing_from(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, "copy", "/a", 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: 'from' argument missing for 'copy' operation",
+        )
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch({}, "move", "/a", 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_patch: 'from' argument missing for 'move' operation",
+        )
+
+    def test_patch_add_to_dict_binary(self):
+        result = self.json_patch(b'{"b": 2}', "add", "/a", 1)
+        self.assertEqual(result, {"a": 1, "b": 2})
+        result = self.json_patch(bytearray(b'{"b": 2}'), "add", "/a", 1)
+        self.assertEqual(result, {"a": 1, "b": 2})
+
+    # json_patch_recipe
+
+    def test_patch_recipe_process(self):
+        result = self.json_patch_recipe(
+            {},
+            [
+                {"op": "add", "path": "/foo", "value": 1},
+                {"op": "add", "path": "/bar", "value": []},
+                {"op": "add", "path": "/bar/-", "value": 2},
+                {"op": "add", "path": "/bar/0", "value": 1},
+                {"op": "remove", "path": "/bar/0"},
+                {"op": "move", "from": "/foo", "path": "/baz"},
+                {"op": "copy", "from": "/baz", "path": "/bax"},
+                {"op": "copy", "from": "/baz", "path": "/bay"},
+                {"op": "replace", "path": "/baz", "value": [10, 20, 30]},
+                {"op": "add", "path": "/foo", "value": 1},
+                {"op": "add", "path": "/foo", "value": 1},
+                {"op": "test", "path": "/baz/1", "value": 20},
+            ],
+        )
+        self.assertEqual(
+            result, {"bar": [2], "bax": 1, "bay": 1, "baz": [10, 20, 30], "foo": 1}
+        )
+
+    def test_patch_recipe_test_fail(self):
+        result = self.json_patch_recipe(
+            {},
+            [
+                {"op": "add", "path": "/bar", "value": []},
+                {"op": "add", "path": "/bar/-", "value": 2},
+                {"op": "test", "path": "/bar/0", "value": 20},
+                {"op": "add", "path": "/bar/0", "value": 1},
+            ],
+        )
+        self.assertIsNone(result)
+
+    def test_patch_recipe_missing_lib(self):
+        with unittest.mock.patch(
+            "ansible_collections.community.general.plugins.filter.json_patch.HAS_LIB",
+            False,
+        ):
+            with self.assertRaises(AnsibleFilterError) as context:
+                self.json_patch_recipe({}, [])
+            self.assertEqual(
+                str(context.exception),
+                "You need to install 'jsonpatch' package prior to running 'json_patch_recipe' filter",
+            )
+
+    def test_patch_recipe_missing_from(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch_recipe({}, [{"op": "copy", "path": "/a"}])
+        self.assertEqual(
+            str(context.exception),
+            "json_patch_recipe: 'from' argument missing for 'copy' operation",
+        )
+
+    def test_patch_recipe_incorrect_type(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch_recipe({}, "copy")
+        self.assertEqual(
+            str(context.exception),
+            "json_patch_recipe: 'operations' needs to be a list",
+        )
+
+    def test_patch_recipe_test_fail_none(self):
+        result = self.json_patch_recipe(
+            {"a": 1, "b": {"c": 2}, "d": 3},
+            [{"op": "test", "path": "/b/c", "value": 99}],
+        )
+        self.assertIsNone(result)
+
+    def test_patch_recipe_test_fail_fail_pos(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch_recipe(
+                {"a": 1, "b": {"c": 2}, "d": 3},
+                [{"op": "test", "path": "/b/c", "value": 99}],
+                True,
+            )
+        self.assertTrue(
+            "json_patch_recipe: test operation failed" in str(context.exception)
+        )
+
+    def test_patch_recipe_test_fail_fail_kw(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_patch_recipe(
+                {"a": 1, "b": {"c": 2}, "d": 3},
+                [{"op": "test", "path": "/b/c", "value": 99}],
+                fail_test=True,
+            )
+        self.assertTrue(
+            "json_patch_recipe: test operation failed" in str(context.exception)
+        )
+
+    # json_diff
+
+    def test_diff_process(self):
+        result = self.json_diff(
+            {"foo": 1, "bar": {"baz": 2}, "baw": [1, 2, 3], "hello": "day"},
+            {
+                "foo": 1,
+                "bar": {"baz": 2},
+                "baw": [1, 3],
+                "baq": {"baz": 2},
+                "hello": "night",
+            },
+        )
+
+        # Sort as the order is unstable
+        self.assertEqual(
+            sorted(result, key=lambda k: k["path"]),
+            [
+                {"op": "add", "path": "/baq", "value": {"baz": 2}},
+                {"op": "remove", "path": "/baw/1"},
+                {"op": "replace", "path": "/hello", "value": "night"},
+            ],
+        )
+
+    def test_diff_missing_lib(self):
+        with unittest.mock.patch(
+            "ansible_collections.community.general.plugins.filter.json_patch.HAS_LIB",
+            False,
+        ):
+            with self.assertRaises(AnsibleFilterError) as context:
+                self.json_diff({}, {})
+            self.assertEqual(
+                str(context.exception),
+                "You need to install 'jsonpatch' package prior to running 'json_diff' filter",
+            )
+
+    def test_diff_arg_checking(self):
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_diff(1, {})
+        self.assertEqual(
+            str(context.exception), "json_diff: input is not dictionary, list or string"
+        )
+        with self.assertRaises(AnsibleFilterError) as context:
+            self.json_diff({}, 1)
+        self.assertEqual(
+            str(context.exception),
+            "json_diff: target is not dictionary, list or string",
+        )

tests/unit/requirements.txt

@@ -59,4 +59,7 @@ python-nomad < 2.0.0 ; python_version <= '3.6'
 python-nomad >= 2.0.0 ; python_version >= '3.7'
 
 # requirement for jenkins_build, jenkins_node, jenkins_plugin modules
-python-jenkins >= 0.4.12
+python-jenkins >= 0.4.12
+
+# requirement for json_patch, json_patch_recipe and json_patch plugins
+jsonpatch