#!/usr/bin/python3 -cimport os, sys; os.execv(os.path.dirname(sys.argv[1]) + "/../common/pywrap", sys.argv)## Copyright (C) 2015 Red Hat, Inc.# SPDX-License-Identifier: LGPL-2.1-or-laterfrom time import sleepimport storagelibimport testlib@testlib.nondestructiveclass TestStorageMdRaid(storagelib.StorageCase):def wait_states(self, states: dict[str, str]):for s in states.keys():with self.browser.wait_timeout(30):self.browser.wait_in_text(self.card_row("MDRAID device", name=s), states[s])def raid_add_disk(self, name: str):self.dialog_open_with_retry(trigger=lambda: self.browser.click(self.card_button("MDRAID device", "Add disk")),expect=lambda: self.dialog_is_present('disks', name))self.dialog_set_val("disks", {name: True})self.dialog_apply_with_retry()def raid_remove_disk(self, name: str):self.click_dropdown(self.card_row("MDRAID device", name=name), "Remove")def raid_action(self, action: str):self.browser.click(self.card_button("MDRAID device", action))def raid_default_action_start(self, action: str):self.raid_action(action)def raid_default_action_finish(self, action: str):if action == "Stop":# Right after assembling an array the device might be busy# from udev rules probing or the mdadm monitor; retry a# few times. Also, it might come back spontaneously after# having been stopped successfully; wait a bit before# checking whether it is really off.for _ in range(3):try:sleep(10)self.browser.wait_text(self.card_desc("MDRAID device", "State"), "Not running")breakexcept testlib.Error as ex:if not ex.msg.startswith('timeout'):raiseprint("Stopping failed, retrying...")if self.browser.is_present("#dialog"):self.browser.wait_in_text("#dialog", "Error stopping RAID array")self.dialog_cancel()self.dialog_wait_close()self.raid_default_action_start(action)else:self.fail("Timed out waiting for array to get stopped")def raid_default_action(self, action: str):self.raid_default_action_start(action)self.raid_default_action_finish(action)def testRaid(self):m = self.machineb = self.browserself.login_and_go("/storage")# Add four disks and make a RAID out of three of themdisk1 = self.add_loopback_disk(name="loop4")b.wait_visible(self.card_row("Storage", name=disk1))disk2 = self.add_loopback_disk(name="loop5")b.wait_visible(self.card_row("Storage", name=disk2))disk3 = self.add_loopback_disk(name="loop6")b.wait_visible(self.card_row("Storage", name=disk3))disk4 = self.add_loopback_disk(name="loop7")b.wait_visible(self.card_row("Storage", name=disk4))self.click_devices_dropdown('Create MDRAID device')self.dialog_wait_open()self.dialog_wait_val("name", "raid0")self.dialog_wait_val("level", "raid5")self.dialog_apply()self.dialog_wait_error("disks", "At least 2 disks are needed")self.dialog_set_val("disks", {disk1: True})self.dialog_apply()self.dialog_wait_error("disks", "At least 2 disks are needed")self.dialog_set_val("disks", {disk2: True, disk3: True})self.dialog_set_val("level", "raid6")self.dialog_apply()self.dialog_wait_error("disks", "At least 4 disks are needed")self.dialog_set_val("level", "raid5")self.dialog_set_val("name", "raid 0")self.dialog_apply()self.dialog_wait_error("name", "Name cannot contain whitespace.")self.dialog_set_val("name", "raid0")self.dialog_apply()self.dialog_wait_close()b.wait_visible(self.card_row("Storage", name="/dev/md/raid0"))self.addCleanup(m.execute, "if [ -e /dev/md/raid0 ]; then mdadm --stop /dev/md/raid0; fi")self.addCleanup(m.execute, "mount | grep ^/dev/md | cut -f1 -d' ' | xargs -r umount")# Check that Cockpit suggest name that does not yet existsself.click_devices_dropdown('Create MDRAID device')b.wait(lambda: self.dialog_val("name") == "raid1")self.dialog_cancel()self.dialog_wait_close()self.click_card_row("Storage", name="/dev/md/raid0")self.wait_states({disk1: "In sync",disk2: "In sync",disk3: "In sync"})def wait_degraded_state(is_degraded: bool):degraded_selector = ".pf-v6-c-alert h4"if is_degraded:b.wait_in_text(degraded_selector, 'The MDRAID device is in a degraded state')else:b.wait_not_present(degraded_selector)# The preferred device should be /dev/md/raid0, but a freshly# created array seems to have no symlink in /dev/md/.## See https://bugzilla.redhat.com/show_bug.cgi?id=1397320#dev = b.text(self.card_desc("MDRAID device", "Device"))# Degrade and repair itm.execute(f"mdadm --quiet {dev} --fail {disk1}; udevadm trigger")wait_degraded_state(is_degraded=True)self.wait_states({disk1: "Failed"})self.raid_remove_disk(disk1)b.wait_not_present(self.card_row("Disks", name=disk1))# HACK - an additional udev trigger seems necessary for# reliability.#m.execute("udevadm trigger; udevadm settle")self.raid_add_disk(disk1)self.wait_states({disk1: "In sync",disk2: "In sync",disk3: "In sync"})wait_degraded_state(is_degraded=False)# Turn it off and on againwith b.wait_timeout(30):self.raid_default_action("Stop")b.wait_text(self.card_desc("MDRAID device", "State"), "Not running")self.raid_default_action("Start")b.wait_text(self.card_desc("MDRAID device", "State"), "Running")# Stopping and starting the array should not change the device.#b.wait_text(self.card_desc("MDRAID device", "Device"), "/dev/md/raid0")# Partitions also get symlinks in /dev/md/, so they should# have names like "/dev/md/raid0p1".#part_prefix = "md/raid0p"# Create Partition Tableself.click_card_dropdown("MDRAID device", "Create partition table")self.dialog({"type": "gpt"})b.wait_text(self.card_row_col("GPT partitions", 1, 1), "Free space")# Create first partitionself.click_dropdown(self.card_row("GPT partitions", 1), "Create partition")self.dialog({"size": 20,"type": "ext4","mount_point": f"{self.mnt_dir}/foo1","name": "One"},secondary=True)b.wait_text(self.card_row_col("GPT partitions", 1, 2), "ext4 filesystem")b.wait_text(self.card_row_col("GPT partitions", 1, 1), part_prefix + "1")# Create second partitionself.click_dropdown(self.card_row("GPT partitions", 2), "Create partition")self.dialog({"type": "ext4","mount_point": f"{self.mnt_dir}/foo2","name": "Two"})b.wait_text(self.card_row_col("GPT partitions", 2, 2), "ext4 filesystem")b.wait_text(self.card_row_col("GPT partitions", 2, 1), part_prefix + "2")b.wait_not_present(self.card_row("GPT partitions", name="Free space"))b.wait_visible(self.card_row("GPT partitions", location=f"{self.mnt_dir}/foo2") + " .usage-bar[role=progressbar]")b.assert_pixels('body', "page",ignore=[self.card_desc("MDRAID device", "UUID")])# Replace a disk by adding a spare and then removing a "In sync" disk# Add a spareself.raid_add_disk(disk4)self.wait_states({disk4: "Spare"})# Remove disk1. The spare takes over.self.raid_remove_disk(disk1)try:b.wait_not_present(self.card_row("MDRAID device", name=disk1))except testlib.Error:# Sometimes disk1 is busy. Try again in that caseif b.is_visible("#dialog") and "Device or resource busy" in b.text("#dialog"):self.dialog_cancel()self.dialog_wait_close()self.raid_remove_disk(disk1)b.wait_not_present(self.card_row("MDRAID device", name=disk1))else:raiseself.wait_states({disk4: "In sync"})# Stop the array, destroy a disk, and start the arrayself.raid_default_action_start("Stop")self.dialog_wait_open()b.wait_in_text('#dialog', "unmount, stop")b.assert_pixels('#dialog', "stop-busy")self.dialog_apply()self.dialog_wait_close()with b.wait_timeout(60):self.raid_default_action_finish("Stop")b.wait_text(self.card_desc("MDRAID device", "State"), "Not running")m.execute(f"wipefs -a {disk2}")b.wait_not_present(self.card_row("MDRAID device", name=disk2))self.raid_default_action("Start")b.wait_text(self.card_desc("MDRAID device", "State"), "Running")wait_degraded_state(is_degraded=True)# Add disk. The array recovers.self.raid_add_disk(disk1)self.wait_states({disk1: "In sync"})wait_degraded_state(is_degraded=False)# Add disk again, as a spareself.raid_add_disk(disk2)self.wait_states({disk2: "Spare"})# Remove it by formatting disk2self.click_card_row("MDRAID device", name=disk2)self.click_card_dropdown("Block device", "Create partition table")b.wait_in_text('#dialog', "remove from MDRAID, initialize")self.dialog_set_val("type", "empty")self.dialog_apply()self.dialog_wait_close()b.go("#/")self.click_card_row("Storage", name="/dev/md/raid0")b.wait_visible(self.card("MDRAID device"))b.wait_not_present(self.card_row("MDRAID device", name=disk2))# Delete the array. We are back on the storage page.self.click_card_dropdown("MDRAID device", "Delete")self.confirm()with b.wait_timeout(120):b.wait_visible(self.card("Storage"))b.wait_not_present(self.card_row("Storage", name="/dev/md/raid0"))def testNotRemovingDisks(self):m = self.machineb = self.browserself.login_and_go("/storage")disk1 = self.add_loopback_disk()b.wait_visible(self.card_row("Storage", name=disk1))disk2 = self.add_loopback_disk()b.wait_visible(self.card_row("Storage", name=disk2))disk3 = self.add_loopback_disk()b.wait_visible(self.card_row("Storage", name=disk3))self.click_devices_dropdown('Create MDRAID device')self.dialog_wait_open()self.dialog_set_val("level", "raid5")self.dialog_set_val("disks", {disk1: True, disk2: True})self.dialog_set_val("name", "ARR")self.dialog_apply()self.dialog_wait_close()self.addCleanup(m.execute, "mdadm --stop /dev/md/ARR")self.click_card_row("Storage", name="/dev/md/ARR")self.wait_states({disk1: "In sync",disk2: "In sync"})# All buttons should be disabled when the array is stoppedwith b.wait_timeout(60):self.raid_default_action("Stop")b.wait_text(self.card_desc("MDRAID device", "State"), "Not running")b.wait_visible(self.card_button("MDRAID device", "Add disk") + ":disabled")self.check_dropdown_action_disabled(self.card_row("MDRAID device", name=disk1), "Remove", "The MDRAID device must be running")self.raid_default_action("Start")b.wait_text(self.card_desc("MDRAID device", "State"), "Running")# With a running array, we can add spares, but not remove "in-sync" disksb.wait_not_present(self.card_button("MDRAID device", "Add disk") + ":disabled")self.check_dropdown_action_disabled(self.card_row("MDRAID device", name=disk1), "Remove", "Need a spare disk")# Adding a spare will allow removal of the "in-sync" disks.self.raid_add_disk(disk3)self.wait_states({disk3: "Spare"})self.raid_remove_disk(disk1)self.wait_states({disk3: "In sync",disk2: "In sync"})# Removing the disk will make the rest un-removable againself.check_dropdown_action_disabled(self.card_row("MDRAID device", name=disk3), "Remove", "Need a spare disk")# A failed disk can be removeddev = b.text(self.card_desc("MDRAID device", "Device"))m.execute(f"mdadm --quiet {dev} --fail {disk3}")self.wait_states({disk3: "Failed"})self.raid_remove_disk(disk3)b.wait_not_present(self.card_row("MDRAID device", name=disk3))# The last disk can not be removedself.check_dropdown_action_disabled(self.card_row("MDRAID device", name=disk2), "Remove", "Need a spare disk")def testBitmap(self):m = self.machineb = self.browserself.login_and_go("/storage")# Make three huge block devices, so that we can make an array# that is beyond the threshold where Cockpit and mdadm start# to worry about bitmaps. The backing files are sparse, so# this is okay as long as nobody actually writes a lot to# these devices. (The fourth is used for a journal.)dev1 = self.add_loopback_disk(110000)dev2 = self.add_loopback_disk(110000)dev3 = self.add_loopback_disk(110000)dev4 = self.add_loopback_disk(100)# We need to use "--assume-clean" so that mdraid doesn't try# to write 110GB to one of the devices during synchronization.m.execute(f"mdadm --create md0 --level=1 --assume-clean --run --raid-devices=2 {dev1} {dev2}")m.execute("udevadm trigger")self.addCleanup(m.execute, "echo /dev/md/* | xargs mdadm --stop $d")self.click_card_row("Storage", name="/dev/md/md0")self.wait_states({dev1: "In sync",dev2: "In sync"})if "Consistency Policy : bitmap" in m.execute("mdadm --misc -D /dev/md/md0"):# Earlier versions of mdadm automatically add a bitmap for# large arrays. Check that Cockpit doesn't complain and# then remove the bitmap to provoke the alert.b.wait_not_present('.pf-v6-c-alert:contains("This MDRAID device has no write-intent bitmap")')m.execute("mdadm --grow --bitmap=none /dev/md/md0; udevadm trigger /dev/md/md0")self.assertNotIn("bitmap", m.execute("mdadm --misc -D /dev/md/md0"))b.wait_in_text('.pf-v6-c-alert', 'This MDRAID device has no write-intent bitmap')b.click('button:contains("Add a bitmap")')b.wait_not_present('.pf-v6-c-alert:contains("This MDRAID device has no write-intent bitmap")')self.assertIn("Consistency Policy : bitmap", m.execute("mdadm --misc -D /dev/md/md0"))# Delete the device and create a new one with a journalself.click_card_dropdown("MDRAID device", "Delete")self.confirm()b.wait_visible(self.card("Storage"))b.wait_not_present(self.card_row("Storage", name="/dev/md/raid0"))m.execute(f"mdadm --create md1 --level=5 --assume-clean --run --raid-devices=3 {dev1} {dev2} {dev3} --write-journal {dev4}")m.execute("udevadm trigger")self.click_card_row("Storage", name="/dev/md/md1")self.wait_states({dev1: "In sync",dev2: "In sync",dev3: "In sync",dev4: "Unknown (journal)"})# There should be no bitmap (regardless of mdadm version), and cockpit should not complainself.assertNotIn("bitmap", m.execute("mdadm --misc -D /dev/md/md1"))b.wait_not_present('.pf-v6-c-alert:contains("This MDRAID device has no write-intent bitmap")')if __name__ == '__main__':testlib.test_main()