tst-{pthread|sched}-affinity-inheritance: Only use in-mask CPUs in affinity mask.

On s390x, the two tests are failing on a lpar with linux 6.18 booted with nosmt:
FAIL:  misc/tst-sched-affinity-inheritance
FAIL:  nptl/tst-pthread-affinity-inheritance

Both tests were failing because they are claiming that the CPU affinity mask was
not correctly inherited in the forked process/thread.
The error already happened before forking, as the mask was set up to use
CPUs 0-14 in set_cpu_mask().  But according to e.g. lscpu or the line
"Cpus_allowed_list" in /proc/self/status, only the following CPUs were in mask at startup:
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
while 1, 3, 5, ... were configured, but not online.

Now the test first gets the current CPU mask at startup in do_test() and
set_cpu_mask() only selects those available CPUs.  The test is now just
counting the available CPUs instead of relying on get_nprocs().  This allows
to run the test even if it was already limited via taskset.

Therefore both concrete tests now have the get_my_affinity() function.  This
also allows to implement verify_my_affinity() in the common skeleton.

In order to ensure that the affinity mask is really set as expected,
verify_my_affinity() is now also called before forking a new process or
creating a new thread.

For also printing the list of mismatched CPU IDs in xor_set in
verify_my_affinity(), the argument order of CPU_ISSET_S is now fixed.
Furthermore we now keep iterating over all configured CPUs to print
the list of cpucount IDs instead of iterating over the first nproc CPUs.
Reviewed-by: Florian Weimer <fweimer@redhat.com>

nptl/tst-pthread-affinity-inheritance.c

@@ -24,49 +24,20 @@
 #include <string.h>
 #include <support/check.h>
 
+static void
+get_my_affinity (size_t size, cpu_set_t *set)
+{
+  int ret = pthread_getaffinity_np (pthread_self (), size, set);
+  if (ret != 0)
+    FAIL ("pthread_getaffinity_np returned %d (%s)", ret, strerror (ret));
+}
+
 static void
 set_my_affinity (size_t size, const cpu_set_t *set)
 {
   int ret = pthread_setaffinity_np (pthread_self (), size, set);
-
   if (ret != 0)
     FAIL ("pthread_setaffinity_np returned %d (%s)", ret, strerror (ret));
 }
 
-static void
-verify_my_affinity (int nproc, int nproc_configured, size_t size,
-		    const cpu_set_t *expected_set)
-{
-  cpu_set_t *set = CPU_ALLOC (nproc_configured);
-  cpu_set_t *xor_set = CPU_ALLOC (nproc_configured);
-
-  if (set == NULL || xor_set== NULL)
-    FAIL_EXIT1 ("verify_my_affinity: Failed to allocate cpuset: %m\n");
-
-  int ret = pthread_getaffinity_np (pthread_self (), size, set);
-  if (ret != 0)
-    FAIL ("pthread_getaffinity_np returned %d (%s)", ret, strerror (ret));
-
-  CPU_XOR_S (size, xor_set, expected_set, set);
-
-  int cpucount = CPU_COUNT_S (size, xor_set);
-
-  if (cpucount > 0)
-    {
-      FAIL ("Affinity mask not inherited, "
-	    "following %d CPUs mismatched in the expected and actual sets: ",
-	    cpucount);
-      for (int cur = 0; cur < nproc && cpucount >= 0; cur++)
-	if (CPU_ISSET_S (size, cur, xor_set))
-	  {
-	    printf ("%d ", cur);
-	    cpucount--;
-	  }
-      printf ("\n");
-    }
-
-  CPU_FREE (set);
-  CPU_FREE (xor_set);
-}
-
 #include "tst-skeleton-affinity-inheritance.c"

nptl/tst-skeleton-affinity-inheritance.c

@@ -42,9 +42,9 @@
 
 struct test_param
 {
-  int nproc;
   int nproc_configured;
   cpu_set_t *set;
+  cpu_set_t *set_at_startup;
   size_t size;
   bool entry;
 };
@@ -52,18 +52,61 @@ struct test_param
 void __attribute__((noinline))
 set_cpu_mask (struct test_param *param, bool entry)
 {
-  int cpus = param->nproc;
+  /* Do not determine the number of CPUs based on get_nproc as the test process
+     could already be taskset to less CPUs.  */
+  int cpus = CPU_COUNT_S (param->size, param->set_at_startup);
 
   /* Less CPUS for the first level, if that's possible.  */
   if (entry && cpus > 1)
     cpus--;
 
+  /* Only use CPUs which were in mask at start of the test process.  */
   CPU_ZERO_S (param->size, param->set);
-  while (cpus > 0)
-    CPU_SET_S (--cpus, param->size, param->set);
+  for (int cur = 0; cpus > 0; cur++)
+    {
+      if (CPU_ISSET_S (cur, param->size, param->set_at_startup))
+	{
+	  CPU_SET_S (cur, param->size, param->set);
+	  cpus--;
+	}
+    }
 
-  if (CPU_COUNT_S (param->size, param->set) == 0)
-    FAIL_EXIT1 ("Failed to add any CPUs to the affinity set\n");
+  if (cpus != 0)
+    FAIL_EXIT1 ("Failed to add all requested CPUs to the affinity set\n");
+}
+
+static void
+verify_my_affinity (int nproc_configured, size_t size,
+		    const cpu_set_t *expected_set)
+{
+  cpu_set_t *set = CPU_ALLOC (nproc_configured);
+  cpu_set_t *xor_set = CPU_ALLOC (nproc_configured);
+
+  if (set == NULL || xor_set == NULL)
+    FAIL_EXIT1 ("verify_my_affinity: Failed to allocate cpuset: %m\n");
+
+  get_my_affinity (size, set);
+
+  CPU_XOR_S (size, xor_set, expected_set, set);
+
+  int cpucount = CPU_COUNT_S (size, xor_set);
+
+  if (cpucount > 0)
+    {
+      FAIL ("Affinity mask not inherited, "
+	    "following %d CPUs mismatched in the expected and actual sets:\n",
+	    cpucount);
+      for (int cur = 0; cpucount > 0; cur++)
+	if (CPU_ISSET_S (cur, size, xor_set))
+	  {
+	    printf ("%d ", cur);
+	    cpucount--;
+	  }
+      printf ("\n");
+    }
+
+  CPU_FREE (set);
+  CPU_FREE (xor_set);
 }
 
 static void *
@@ -72,8 +115,7 @@ child_test (void *arg)
   struct test_param *param = arg;
 
   printf ("%d:%d        child\n", getpid (), gettid ());
-  verify_my_affinity (param->nproc, param->nproc_configured, param->size,
-		      param->set);
+  verify_my_affinity (param->nproc_configured, param->size, param->set);
   return NULL;
 }
 
@@ -91,13 +133,15 @@ do_one_test (void *arg)
       child = do_one_test;
       set_cpu_mask (param, true);
       set_my_affinity (param->size, param->set);
+      /* Ensure that the affinity mask is really set as expected before forking
+	 a new process or creating a new thread.  */
+      verify_my_affinity (param->nproc_configured, param->size, param->set);
       param->entry = false;
     }
   else
     {
       /* Verification for the first level.  */
-      verify_my_affinity (param->nproc, param->nproc_configured, param->size,
-			  param->set);
+      verify_my_affinity (param->nproc_configured, param->size, param->set);
 
       /* Launch the second level test, launching CHILD_TEST as a subprocess and
 	 then as a subthread.  Use a different mask to see if it gets
@@ -136,23 +180,30 @@ do_test (void)
   /* Large enough in case the kernel decides to return the larger mask.  This
      seems to happen on some kernels for S390x.  */
   int num_configured_cpus = get_nprocs_conf ();
-  int num_cpus = get_nprocs ();
 
   struct test_param param =
     {
-      .nproc = num_cpus,
       .nproc_configured = num_configured_cpus,
       .set = CPU_ALLOC (num_configured_cpus),
+      .set_at_startup = CPU_ALLOC (num_configured_cpus),
       .size = CPU_ALLOC_SIZE (num_configured_cpus),
       .entry = true,
     };
 
   if (param.set == NULL)
-    FAIL_EXIT1 ("error: CPU_ALLOC (%d) failed\n", num_cpus);
+    FAIL_EXIT1 ("error: CPU_ALLOC (%d) for param.set failed\n",
+		num_configured_cpus);
+  if (param.set_at_startup == NULL)
+    FAIL_EXIT1 ("error: CPU_ALLOC (%d) for param.set_at_startup failed\n",
+		num_configured_cpus);
+
+  /* Get a list of all available CPUs for this test.  */
+  get_my_affinity (param.size, param.set_at_startup);
 
   do_one_test (&param);
 
   CPU_FREE (param.set);
+  CPU_FREE (param.set_at_startup);
 
   return 0;
 }

sysdeps/unix/sysv/linux/tst-sched-affinity-inheritance.c

@@ -24,6 +24,14 @@
 #include <stdio.h>
 #include <support/check.h>
 
+static void
+get_my_affinity (size_t size, cpu_set_t *set)
+{
+  int ret = sched_getaffinity (0, size, set);
+  if (ret != 0)
+    FAIL ("sched_getaffinity returned %d (%s)", ret, strerror (ret));
+}
+
 static void
 set_my_affinity (size_t size, const cpu_set_t *set)
 {
@@ -33,40 +41,4 @@ set_my_affinity (size_t size, const cpu_set_t *set)
     FAIL ("sched_setaffinity returned %d (%s)", ret, strerror (ret));
 }
 
-static void
-verify_my_affinity (int nproc, int nproc_configured, size_t size,
-		    const cpu_set_t *expected_set)
-{
-  cpu_set_t *set = CPU_ALLOC (nproc_configured);
-  cpu_set_t *xor_set = CPU_ALLOC (nproc_configured);
-
-  if (set == NULL || xor_set== NULL)
-    FAIL_EXIT1 ("verify_my_affinity: Failed to allocate cpuset: %m\n");
-
-  int ret = sched_getaffinity (0, size, set);
-  if (ret != 0)
-    FAIL ("sched_getaffinity returned %d (%s)", ret, strerror (ret));
-
-  CPU_XOR_S (size, xor_set, expected_set, set);
-
-  int cpucount = CPU_COUNT_S (size, xor_set);
-
-  if (cpucount > 0)
-    {
-      FAIL ("Affinity mask not inherited, "
-	    "following %d CPUs mismatched in the expected and actual sets:\n",
-	    cpucount);
-      for (int cur = 0; cur < nproc && cpucount >= 0; cur++)
-	if (CPU_ISSET_S (size, cur, xor_set))
-	  {
-	    printf ("%d ", cur);
-	    cpucount--;
-	  }
-      printf ("\n");
-    }
-
-  CPU_FREE (set);
-  CPU_FREE (xor_set);
-}
-
 #include <nptl/tst-skeleton-affinity-inheritance.c>