s390/ap: rearm APQNs bindings complete completion

The APQN bindings complete completion was used to reflect
that 1st the AP bus initial scan is done and 2nd all the
detected APQNs have been bound to a device driver.
This was a single-shot action. However, as the AP bus
supports hot-plug it may be that new APQNs appear reflected
as new AP queue and card devices which need to be bound
to appropriate device drivers. So the condition that
all existing AP queue devices are bound to device drivers
may go away for a certain time.

This patch now checks during AP bus scan for maybe new AP
devices appearing and does a re-init of the internal completion
variable. So the AP bus function ap_wait_apqn_bindings_complete()
now may block on this condition variable even later after
initial scan is through when new APQNs appear which need to
get bound.

This patch also moves the check for binding complete invocation
from the probe function to the end of the AP bus scan function.
This change also covers some weird scenarios where during a
card hotplug the binding of the card device was sufficient for
binding complete but the queue devices where still in the
process of being discovered.

As of now this change has no impact on existing code. The
behavior change in the now later bindings complete should not
impact any code (and has been tested so far). The only
exploiter is the zcrypt function zcrypt_wait_api_operational()
which only initial calls ap_wait_apqn_bindings_complete().

However, this new behavior of the AP bus wait for APQNs bindings
complete function will be used in a later patch exploiting
this for the zcrypt API layer.

Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
Reviewed-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

3 files changed, 80 insertions, 21 deletions

diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index a58190e4b745..56f32f732ddf 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -90,8 +90,8 @@ static atomic64_t ap_scan_bus_count;
 /* # of bindings complete since init */
 static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0);
 
-/* completion for initial APQN bindings complete */
-static DECLARE_COMPLETION(ap_init_apqn_bindings_complete);
+/* completion for APQN bindings complete */
+static DECLARE_COMPLETION(ap_apqn_bindings_complete);
 
 static struct ap_config_info *ap_qci_info;
 static struct ap_config_info *ap_qci_info_old;
@@ -754,7 +754,7 @@ static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound)
 }
 
 /*
- * After initial ap bus scan do check if all existing APQNs are
+ * After ap bus scan do check if all existing APQNs are
  * bound to device drivers.
  */
 static void ap_check_bindings_complete(void)
@@ -764,9 +764,9 @@ static void ap_check_bindings_complete(void)
 	if (atomic64_read(&ap_scan_bus_count) >= 1) {
 		ap_calc_bound_apqns(&apqns, &bound);
 		if (bound == apqns) {
-			if (!completion_done(&ap_init_apqn_bindings_complete)) {
-				complete_all(&ap_init_apqn_bindings_complete);
-				AP_DBF_INFO("%s complete\n", __func__);
+			if (!completion_done(&ap_apqn_bindings_complete)) {
+				complete_all(&ap_apqn_bindings_complete);
+				pr_debug("%s all apqn bindings complete\n", __func__);
 			}
 			ap_send_bindings_complete_uevent();
 		}
@@ -783,27 +783,29 @@ static void ap_check_bindings_complete(void)
  * -ETIME is returned. On failures negative return values are
  * returned to the caller.
  */
-int ap_wait_init_apqn_bindings_complete(unsigned long timeout)
+int ap_wait_apqn_bindings_complete(unsigned long timeout)
 {
+	int rc = 0;
 	long l;
 
-	if (completion_done(&ap_init_apqn_bindings_complete))
+	if (completion_done(&ap_apqn_bindings_complete))
 		return 0;
 
 	if (timeout)
 		l = wait_for_completion_interruptible_timeout(
-			&ap_init_apqn_bindings_complete, timeout);
+			&ap_apqn_bindings_complete, timeout);
 	else
 		l = wait_for_completion_interruptible(
-			&ap_init_apqn_bindings_complete);
+			&ap_apqn_bindings_complete);
 	if (l < 0)
-		return l == -ERESTARTSYS ? -EINTR : l;
+		rc = l == -ERESTARTSYS ? -EINTR : l;
 	else if (l == 0 && timeout)
-		return -ETIME;
+		rc = -ETIME;
 
-	return 0;
+	pr_debug("%s rc=%d\n", __func__, rc);
+	return rc;
 }
-EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete);
+EXPORT_SYMBOL(ap_wait_apqn_bindings_complete);
 
 static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data)
 {
@@ -940,8 +942,6 @@ static int ap_device_probe(struct device *dev)
 		if (is_queue_dev(dev))
 			hash_del(&to_ap_queue(dev)->hnode);
 		spin_unlock_bh(&ap_queues_lock);
-	} else {
-		ap_check_bindings_complete();
 	}
 
 out:
@@ -2136,6 +2136,49 @@ static bool ap_get_configuration(void)
 		      sizeof(struct ap_config_info)) != 0;
 }
 
+/*
+ * ap_config_has_new_aps - Check current against old qci info if
+ * new adapters have appeared. Returns true if at least one new
+ * adapter in the apm mask is showing up. Existing adapters or
+ * receding adapters are not counted.
+ */
+static bool ap_config_has_new_aps(void)
+{
+
+	unsigned long m[BITS_TO_LONGS(AP_DEVICES)];
+
+	if (!ap_qci_info)
+		return false;
+
+	bitmap_andnot(m, (unsigned long *)ap_qci_info->apm,
+		      (unsigned long *)ap_qci_info_old->apm, AP_DEVICES);
+	if (!bitmap_empty(m, AP_DEVICES))
+		return true;
+
+	return false;
+}
+
+/*
+ * ap_config_has_new_doms - Check current against old qci info if
+ * new (usage) domains have appeared. Returns true if at least one
+ * new domain in the aqm mask is showing up. Existing domains or
+ * receding domains are not counted.
+ */
+static bool ap_config_has_new_doms(void)
+{
+	unsigned long m[BITS_TO_LONGS(AP_DOMAINS)];
+
+	if (!ap_qci_info)
+		return false;
+
+	bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm,
+		      (unsigned long *)ap_qci_info_old->aqm, AP_DOMAINS);
+	if (!bitmap_empty(m, AP_DOMAINS))
+		return true;
+
+	return false;
+}
+
 /**
  * ap_scan_bus(): Scan the AP bus for new devices
  * Runs periodically, workqueue timer (ap_config_time)
@@ -2147,10 +2190,21 @@ static void ap_scan_bus(struct work_struct *unused)
 
 	pr_debug(">%s\n", __func__);
 
-	/* config change notify */
+	/* (re-)fetch configuration via QCI */
 	config_changed = ap_get_configuration();
-	if (config_changed)
+	if (config_changed) {
+		if (ap_config_has_new_aps() || ap_config_has_new_doms()) {
+			/*
+			 * Appearance of new adapters and/or domains need to
+			 * build new ap devices which need to get bound to an
+			 * device driver. Thus reset the APQN bindings complete
+			 * completion.
+			 */
+			reinit_completion(&ap_apqn_bindings_complete);
+		}
+		/* post a config change notify */
 		notify_config_changed();
+	}
 	ap_select_domain();
 
 	/* loop over all possible adapters */
@@ -2177,9 +2231,10 @@ static void ap_scan_bus(struct work_struct *unused)
 	if (atomic64_inc_return(&ap_scan_bus_count) == 1) {
 		pr_debug("%s init scan complete\n", __func__);
 		ap_send_init_scan_done_uevent();
-		ap_check_bindings_complete();
 	}
 
+	ap_check_bindings_complete();
+
 	mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ);
 
 	pr_debug("<%s\n", __func__);
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 98814839ef30..be320bd46f65 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -352,8 +352,12 @@ int ap_parse_mask_str(const char *str,
  * the return value is 0. If the timeout (in jiffies) hits instead
  * -ETIME is returned. On failures negative return values are
  * returned to the caller.
+ * It may be that the AP bus scan finds new devices. Then the
+ * condition that all APQNs are bound to their device drivers
+ * is reset to false and this call again blocks until either all
+ * APQNs are bound to a device driver or the timeout hits again.
  */
-int ap_wait_init_apqn_bindings_complete(unsigned long timeout);
+int ap_wait_apqn_bindings_complete(unsigned long timeout);
 
 void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg);
 void ap_send_online_uevent(struct ap_device *ap_dev, int online);
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
index e8742757085b..52990a8553e0 100644
--- a/drivers/s390/crypto/zcrypt_api.c
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -2008,7 +2008,7 @@ int zcrypt_wait_api_operational(void)
 	switch (zcrypt_wait_api_state) {
 	case 0:
 		/* initial state, invoke wait for the ap bus complete */
-		rc = ap_wait_init_apqn_bindings_complete(
+		rc = ap_wait_apqn_bindings_complete(
 			msecs_to_jiffies(60 * 1000));
 		switch (rc) {
 		case 0: