filesystem: revamp module (#2472)

* revamp filesystem module to prepare next steps

* pass all commands to module.run_command() as lists
* refactor grow() and grow_cmd() to not need to override them so much
* refactor all existing get_fs_size() overrides to raise a ValueError if
  not able to parse command output and return an integer.
* override MKFS_FORCE_FLAGS the same way for all fstypes that require it
* improve documentation of limitations of the module regarding FreeBSD
* fix indentation in DOCUMENTATION
* add/update function/method docstrings
* fix pylint hints

filesystem: refactor integration tests

* Include *reiserfs* and *swap* in tests.
* Fix reiserfs related code and tests accordingly.
* Replace "other fs" (unhandled by this module), from *swap* to *minix*
  (both mkswap and mkfs.minix being provided by util-linux).
* Replace *dd* commands by *filesize* dedicated module.
* Use FQCNs and name the tasks.
* Update main tests conditionals.

* add a changelog fragment

* Apply suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>

* declare variables as lists when lists are needed

* fix construction without useless conversion

Co-authored-by: Felix Fontein <felix@fontein.de>
This commit is contained in:
quidame 2021-05-18 06:46:45 +02:00 committed by GitHub
commit f6db0745fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 434 additions and 288 deletions

View file

@ -17,7 +17,9 @@ tested_filesystems:
ext2: {fssize: 10, grow: True}
xfs: {fssize: 20, grow: False} # grow requires a mounted filesystem
btrfs: {fssize: 150, grow: False} # grow not implemented
reiserfs: {fssize: 33, grow: False} # grow not implemented
vfat: {fssize: 20, grow: True}
ocfs2: {fssize: '{{ ocfs2_fssize }}', grow: False} # grow not implemented
f2fs: {fssize: '{{ f2fs_fssize|default(60) }}', grow: 'f2fs_version is version("1.10.0", ">=")'}
lvm: {fssize: 20, grow: True}
swap: {fssize: 10, grow: False} # grow not implemented

View file

@ -1,6 +1,9 @@
---
- name: 'Create a "disk" file'
command: 'dd if=/dev/zero of={{ image_file }} bs=1M count={{ fssize }}'
community.general.filesize:
path: '{{ image_file }}'
size: '{{ fssize }}M'
force: true
- vars:
dev: '{{ image_file }}'
@ -8,26 +11,29 @@
- when: fstype == 'lvm'
block:
- name: 'Create a loop device for LVM'
command: 'losetup --show -f {{ dev }}'
ansible.builtin.command:
cmd: 'losetup --show -f {{ dev }}'
register: loop_device_cmd
- set_fact:
- name: 'Switch to loop device target for further tasks'
ansible.builtin.set_fact:
dev: "{{ loop_device_cmd.stdout }}"
- include_tasks: '{{ action }}.yml'
always:
- name: 'Detach loop device used for LVM'
command: 'losetup -d {{ dev }}'
args:
ansible.builtin.command:
cmd: 'losetup -d {{ dev }}'
removes: '{{ dev }}'
when: fstype == 'lvm'
- name: 'Clean correct device for LVM'
set_fact:
ansible.builtin.set_fact:
dev: '{{ image_file }}'
when: fstype == 'lvm'
- file:
- name: 'Remove disk image file'
ansible.builtin.file:
name: '{{ image_file }}'
state: absent

View file

@ -1,43 +1,58 @@
- name: filesystem creation
filesystem:
---
- name: "Create filesystem"
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
register: fs_result
- assert:
- name: "Assert that results are as expected"
ansible.builtin.assert:
that:
- 'fs_result is changed'
- 'fs_result is success'
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: "Get UUID of created filesystem"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid
- name: "Check that filesystem isn't created if force isn't used"
filesystem:
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
register: fs2_result
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: "Get UUID of the filesystem"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid2
- assert:
- name: "Assert that filesystem UUID is not changed"
ansible.builtin.assert:
that:
- 'not (fs2_result is changed)'
- 'fs2_result is not changed'
- 'fs2_result is success'
- 'uuid.stdout == uuid2.stdout'
- name: Check that filesystem is recreated if force is used
filesystem:
- name: "Check that filesystem is recreated if force is used"
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
force: yes
register: fs3_result
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: "Get UUID of the new filesystem"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid3
- assert:
- name: "Assert that filesystem UUID is changed"
# libblkid gets no UUID at all for this fstype on FreeBSD
when: not (ansible_system == 'FreeBSD' and fstype == 'reiserfs')
ansible.builtin.assert:
that:
- 'fs3_result is changed'
- 'fs3_result is success'
@ -46,24 +61,31 @@
- when: 'grow|bool and (fstype != "vfat" or resize_vfat)'
block:
- name: increase fake device
shell: 'dd if=/dev/zero bs=1M count=1 >> {{ image_file }}'
- name: "Increase fake device"
community.general.filesize:
path: '{{ image_file }}'
size: '{{ fssize | int + 1 }}M'
- name: Resize loop device for LVM
command: losetup -c {{ dev }}
- name: "Resize loop device for LVM"
ansible.builtin.command:
cmd: 'losetup -c {{ dev }}'
when: fstype == 'lvm'
- name: Expand filesystem
filesystem:
- name: "Expand filesystem"
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
resizefs: yes
register: fs4_result
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: "Get UUID of the filesystem"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid4
- assert:
- name: "Assert that filesystem UUID is not changed"
ansible.builtin.assert:
that:
- 'fs4_result is changed'
- 'fs4_result is success'
@ -74,14 +96,15 @@
(fstype == "xfs" and ansible_system == "Linux" and
ansible_distribution not in ["CentOS", "Ubuntu"])
block:
- name: Check that resizefs does nothing if device size is not changed
filesystem:
- name: "Check that resizefs does nothing if device size is not changed"
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
resizefs: yes
register: fs5_result
- assert:
- name: "Assert that the state did not change"
ansible.builtin.assert:
that:
- 'fs5_result is not changed'
- 'fs5_result is succeeded'

View file

@ -4,9 +4,9 @@
# and should not be used as examples of how to write Ansible roles #
####################################################################
- debug:
- ansible.builtin.debug:
msg: '{{ role_name }}'
- debug:
- ansible.builtin.debug:
msg: '{{ role_path|basename }}'
- import_tasks: setup.yml
@ -27,29 +27,35 @@
grow: '{{ item.0.value.grow }}'
action: '{{ item.1 }}'
when:
- 'not (item.0.key == "btrfs" and ansible_system == "FreeBSD")' # btrfs not available on FreeBSD
# On Ubuntu trusty, blkid is unable to identify filesystem smaller than 256Mo, see
# https://www.kernel.org/pub/linux/utils/util-linux/v2.21/v2.21-ChangeLog
# https://anonscm.debian.org/cgit/collab-maint/pkg-util-linux.git/commit/?id=04f7020eadf31efc731558df92daa0a1c336c46c
- 'not (item.0.key == "btrfs" and (ansible_distribution == "Ubuntu" and ansible_distribution_release == "trusty"))'
- 'not (item.0.key == "btrfs" and (ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version is version("8", ">=")))'
- 'not (item.0.key == "lvm" and ansible_system == "FreeBSD")' # LVM not available on FreeBSD
- 'not (item.0.key == "lvm" and ansible_virtualization_type in ["docker", "container", "containerd"])' # Tests use losetup which can not be used inside unprivileged container
- 'not (item.0.key == "ocfs2" and ansible_os_family != "Debian")' # ocfs2 only available on Debian based distributions
- 'not (item.0.key == "f2fs" and ansible_system == "FreeBSD")'
# f2fs-tools package not available with RHEL/CentOS
- 'not (item.0.key == "f2fs" and ansible_distribution in ["CentOS", "RedHat"])'
# On Ubuntu trusty, blkid (2.20.1) is unable to identify F2FS filesystem. blkid handles F2FS since v2.23, see:
# https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.23/v2.23-ReleaseNotes
- 'not (item.0.key == "f2fs" and ansible_distribution == "Ubuntu" and ansible_distribution_version is version("14.04", "<="))'
- 'not (item.1 == "overwrite_another_fs" and ansible_system == "FreeBSD")'
# FreeBSD limited support
# Not available: btrfs, lvm, f2fs, ocfs2
# All BSD systems use swap fs, but only Linux needs mkswap
# Supported: ext2/3/4 (e2fsprogs), xfs (xfsprogs), reiserfs (progsreiserfs), vfat
- 'not (ansible_system == "FreeBSD" and item.0.key in ["btrfs", "f2fs", "swap", "lvm", "ocfs2"])'
# Available on FreeBSD but not on testbed (util-linux conflicts with e2fsprogs): wipefs, mkfs.minix
- 'not (ansible_system == "FreeBSD" and item.1 in ["overwrite_another_fs", "remove_fs"])'
# Other limitations and corner cases
# f2fs-tools and reiserfs-utils packages not available with RHEL/CentOS on CI
- 'not (ansible_distribution in ["CentOS", "RedHat"] and item.0.key in ["f2fs", "reiserfs"])'
- 'not (ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", ">=") and
item.0.key == "btrfs")'
# ocfs2 only available on Debian based distributions
- 'not (item.0.key == "ocfs2" and ansible_os_family != "Debian")'
# Tests use losetup which can not be used inside unprivileged container
- 'not (item.0.key == "lvm" and ansible_virtualization_type in ["docker", "container", "containerd"])'
- 'not (item.1 == "remove_fs" and ansible_system == "FreeBSD")' # util-linux not available on FreeBSD
# On CentOS 6 shippable containers, wipefs seems unable to remove vfat signatures
- 'not (item.1 == "remove_fs" and item.0.key == "vfat" and ansible_distribution == "CentOS" and
ansible_distribution_version is version("7.0", "<"))'
- 'not (ansible_distribution == "CentOS" and ansible_distribution_version is version("7.0", "<") and
item.1 == "remove_fs" and item.0.key == "vfat")'
# On same systems, mkfs.minix (unhandled by the module) can't find the device/file
- 'not (ansible_distribution == "CentOS" and ansible_distribution_version is version("7.0", "<") and
item.1 == "overwrite_another_fs")'
# The xfsprogs package on newer versions of OpenSUSE (15+) require Python 3, we skip this on our Python 2 container
# OpenSUSE 42.3 Python2 and the other py3 containers are not affected so we will continue to run that
- 'not (item.0.key == "xfs" and ansible_os_family == "Suse" and ansible_python.version.major == 2 and ansible_distribution_major_version|int != 42)'
- 'not (ansible_os_family == "Suse" and ansible_distribution_major_version|int != 42 and
item.0.key == "xfs" and ansible_python.version.major == 2)'
loop: "{{ query('dict', tested_filesystems)|product(['create_fs', 'overwrite_another_fs', 'remove_fs'])|list }}"

View file

@ -1,40 +1,55 @@
---
- name: 'Recreate "disk" file'
command: 'dd if=/dev/zero of={{ image_file }} bs=1M count={{ fssize }}'
community.general.filesize:
path: '{{ image_file }}'
size: '{{ fssize }}M'
force: true
- name: 'Create a swap filesystem'
command: 'mkswap {{ dev }}'
- name: 'Create a minix filesystem'
ansible.builtin.command:
cmd: 'mkfs.minix {{ dev }}'
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: 'Get UUID of the new filesystem'
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid
- name: "Check that an existing filesystem (not handled by this module) isn't overwritten when force isn't used"
filesystem:
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
register: fs_result
ignore_errors: True
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: 'Get UUID of the filesystem'
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid2
- assert:
- name: 'Assert that module failed and filesystem UUID is not changed'
ansible.builtin.assert:
that:
- 'fs_result is failed'
- 'uuid.stdout == uuid2.stdout'
- name: "Check that an existing filesystem (not handled by this module) is overwritten when force is used"
filesystem:
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
force: yes
register: fs_result2
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
- name: 'Get UUID of the new filesystem'
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: uuid3
- assert:
- name: 'Assert that module succeeded and filesystem UUID is changed'
ansible.builtin.assert:
that:
- 'fs_result2 is successful'
- 'fs_result2 is success'
- 'fs_result2 is changed'
- 'uuid2.stdout != uuid3.stdout'

View file

@ -1,98 +1,98 @@
---
# We assume 'create_fs' tests have passed.
- name: filesystem creation
filesystem:
- name: "Create filesystem"
community.general.filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
- name: get filesystem UUID with 'blkid'
command:
- name: "Get filesystem UUID with 'blkid'"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: blkid_ref
- name: Assert that a filesystem exists on top of the device
assert:
- name: "Assert that a filesystem exists on top of the device"
ansible.builtin.assert:
that:
- blkid_ref.stdout | length > 0
# Test check_mode first
- name: filesystem removal (check mode)
filesystem:
- name: "Remove filesystem (check mode)"
community.general.filesystem:
dev: '{{ dev }}'
state: absent
register: wipefs
check_mode: yes
- name: get filesystem UUID with 'blkid' (should remain the same)
command:
- name: "Get filesystem UUID with 'blkid' (should remain the same)"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
register: blkid
- name: Assert that the state changed but the filesystem still exists
assert:
- name: "Assert that the state changed but the filesystem still exists"
ansible.builtin.assert:
that:
- wipefs is changed
- blkid.stdout == blkid_ref.stdout
# Do it
- name: filesystem removal
filesystem:
- name: "Remove filesystem"
community.general.filesystem:
dev: '{{ dev }}'
state: absent
register: wipefs
- name: get filesystem UUID with 'blkid' (should be empty)
command:
- name: "Get filesystem UUID with 'blkid' (should be empty)"
ansible.builtin.command:
cmd: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
changed_when: false
failed_when: false
register: blkid
- name: Assert that the state changed and the device has no filesystem
assert:
- name: "Assert that the state changed and the device has no filesystem"
ansible.builtin.assert:
that:
- wipefs is changed
- blkid.stdout | length == 0
- blkid.rc == 2
# Do it again
- name: filesystem removal (idempotency)
filesystem:
- name: "Remove filesystem (idempotency)"
community.general.filesystem:
dev: '{{ dev }}'
state: absent
register: wipefs
- name: Assert that the state did not change
assert:
- name: "Assert that the state did not change"
ansible.builtin.assert:
that:
- wipefs is not changed
# and again
- name: filesystem removal (idempotency, check mode)
filesystem:
- name: "Remove filesystem (idempotency, check mode)"
community.general.filesystem:
dev: '{{ dev }}'
state: absent
register: wipefs
check_mode: yes
- name: Assert that the state did not change
assert:
- name: "Assert that the state did not change"
ansible.builtin.assert:
that:
- wipefs is not changed
# By the way, test removal of a filesystem on unexistent device
- name: filesystem removal (unexistent device)
filesystem:
- name: "Remove filesystem (unexistent device)"
community.general.filesystem:
dev: '/dev/unexistent_device'
state: absent
register: wipefs
- name: Assert that the state did not change
assert:
- name: "Assert that the state did not change"
ansible.builtin.assert:
that:
- wipefs is not changed

View file

@ -1,6 +1,9 @@
---
- name: install filesystem tools
package:
# By installing e2fsprogs on FreeBSD, we get a usable blkid command, but this
# package conflicts with util-linux, that provides blkid too, but also wipefs
# (required for filesystem state=absent).
- name: "Install filesystem tools"
ansible.builtin.package:
name: '{{ item }}'
state: present
# xfsprogs on OpenSUSE requires Python 3, skip this for our newer Py2 OpenSUSE builds
@ -9,86 +12,134 @@
- e2fsprogs
- xfsprogs
- block:
- name: install btrfs progs
package:
name: btrfs-progs
state: present
when:
- ansible_os_family != 'Suse'
- not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('16.04', '<='))
- ansible_system != "FreeBSD"
- not (ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version is version('8', '>='))
- name: "Install btrfs progs"
ansible.builtin.package:
name: btrfs-progs
state: present
when:
- ansible_os_family != 'Suse'
- not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('16.04', '<='))
- ansible_system != "FreeBSD"
- not (ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version is version('8', '>='))
- name: install btrfs progs (Ubuntu <= 16.04)
package:
name: btrfs-tools
state: present
when: ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('16.04', '<=')
- name: "Install btrfs tools (Ubuntu <= 16.04)"
ansible.builtin.package:
name: btrfs-tools
state: present
when:
- ansible_distribution == 'Ubuntu'
- ansible_distribution_version is version('16.04', '<=')
- name: install btrfs progs (OpenSuse)
package:
name: '{{ item }}'
state: present
when: ansible_os_family == 'Suse'
with_items:
- python{{ ansible_python.version.major }}-xml
- btrfsprogs
- name: "Install btrfs progs (OpenSuse)"
ansible.builtin.package:
name: '{{ item }}'
state: present
when: ansible_os_family == 'Suse'
with_items:
- python{{ ansible_python.version.major }}-xml
- btrfsprogs
- name: install ocfs2 (Debian)
package:
name: ocfs2-tools
state: present
when: ansible_os_family == 'Debian'
- name: "Install reiserfs utils (Fedora)"
ansible.builtin.package:
name: reiserfs-utils
state: present
when:
- ansible_distribution == 'Fedora'
- when:
- ansible_os_family != 'RedHat' or ansible_distribution == 'Fedora'
- ansible_distribution != 'Ubuntu' or ansible_distribution_version is version('16.04', '>=')
- ansible_system != "FreeBSD"
block:
- name: install f2fs
package:
name: f2fs-tools
state: present
- name: "Install reiserfs (OpenSuse)"
ansible.builtin.package:
name: reiserfs
state: present
when:
- ansible_os_family == 'Suse'
- name: fetch f2fs version
command: mkfs.f2fs /dev/null
ignore_errors: yes
register: mkfs_f2fs
- name: "Install reiserfs progs (Debian and more)"
ansible.builtin.package:
name: reiserfsprogs
state: present
when:
- ansible_system == 'Linux'
- ansible_os_family not in ['Suse', 'RedHat']
- set_fact:
f2fs_version: '{{ mkfs_f2fs.stdout | regex_search("F2FS-tools: mkfs.f2fs Ver:.*") | regex_replace("F2FS-tools: mkfs.f2fs Ver: ([0-9.]+) .*", "\1") }}'
- name: "Install reiserfs progs (FreeBSD)"
ansible.builtin.package:
name: progsreiserfs
state: present
when:
- ansible_system == 'FreeBSD'
- name: install dosfstools and lvm2 (Linux)
package:
name: '{{ item }}'
with_items:
- dosfstools
- lvm2
when: ansible_system == 'Linux'
- name: "Install ocfs2 (Debian)"
ansible.builtin.package:
name: ocfs2-tools
state: present
when: ansible_os_family == 'Debian'
- block:
- name: install fatresize
package:
name: fatresize
state: present
- command: fatresize --help
register: fatresize
- set_fact:
fatresize_version: '{{ fatresize.stdout_lines[0] | regex_search("[0-9]+\.[0-9]+\.[0-9]+") }}'
- name: "Install f2fs tools and get version"
when:
- ansible_os_family != 'RedHat' or ansible_distribution == 'Fedora'
- ansible_distribution != 'Ubuntu' or ansible_distribution_version is version('16.04', '>=')
- ansible_system != "FreeBSD"
block:
- name: "Install f2fs tools"
ansible.builtin.package:
name: f2fs-tools
state: present
- name: "Fetch f2fs version"
ansible.builtin.command:
cmd: mkfs.f2fs /dev/null
changed_when: false
ignore_errors: true
register: mkfs_f2fs
- name: "Record f2fs_version"
ansible.builtin.set_fact:
f2fs_version: '{{ mkfs_f2fs.stdout
| regex_search("F2FS-tools: mkfs.f2fs Ver:.*")
| regex_replace("F2FS-tools: mkfs.f2fs Ver: ([0-9.]+) .*", "\1") }}'
- name: "Install dosfstools and lvm2 (Linux)"
ansible.builtin.package:
name: '{{ item }}'
with_items:
- dosfstools
- lvm2
when: ansible_system == 'Linux'
- name: "Install fatresize and get version"
when:
- ansible_system == 'Linux'
- ansible_os_family != 'Suse'
- ansible_os_family != 'RedHat' or (ansible_distribution == 'CentOS' and ansible_distribution_version is version('7.0', '=='))
block:
- name: "Install fatresize"
ansible.builtin.package:
name: fatresize
state: present
- command: mke2fs -V
- name: "Fetch fatresize version"
ansible.builtin.command:
cmd: fatresize --help
changed_when: false
register: fatresize
- name: "Record fatresize_version"
ansible.builtin.set_fact:
fatresize_version: '{{ fatresize.stdout_lines[0] | regex_search("[0-9]+\.[0-9]+\.[0-9]+") }}'
- name: "Fetch e2fsprogs version"
ansible.builtin.command:
cmd: mke2fs -V
changed_when: false
register: mke2fs
- set_fact:
- name: "Record e2fsprogs_version"
ansible.builtin.set_fact:
# mke2fs 1.43.6 (29-Aug-2017)
e2fsprogs_version: '{{ mke2fs.stderr_lines[0] | regex_search("[0-9]{1,2}\.[0-9]{1,2}(\.[0-9]{1,2})?") }}'
- set_fact:
- name: "Set version-related facts to skip further tasks"
ansible.builtin.set_fact:
# http://e2fsprogs.sourceforge.net/e2fsprogs-release.html#1.43
# Mke2fs no longer complains if the user tries to create a file system
# using the entire block device.