benchtests: Improve benchtests for strstr, memmem, and memchr

1. Use json_ctx for output to help standardize format across all
   benchtests.

2. Add some additional tests to strstr and memchr expanding alignments
   and adding more small values.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>

benchtests/bench-memchr.c

@@ -53,8 +53,11 @@ SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
 }
 #endif /* !USE_AS_MEMRCHR */
 
+#include "json-lib.h"
+
 static void
-do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c,
+	     size_t n)
 {
   size_t i, iters = INNER_LOOP_ITERS_LARGE;
   timing_t start, stop, cur;
@@ -68,15 +71,16 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
+  json_element_double (json_ctx, (double) cur / (double) iters);
 }
 
 static void
-do_test (size_t align, size_t pos, size_t len, int seek_char)
+do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len,
+	 int seek_char)
 {
   size_t i;
 
-  align &= 7;
+  align &= getpagesize () - 1;
   if ((align + len) * sizeof (CHAR) >= page_size)
     return;
 
@@ -100,67 +104,99 @@ do_test (size_t align, size_t pos, size_t len, int seek_char)
       buf[align + len] = seek_char;
     }
 
-  printf ("Length %4zd, position %4zd, alignment %2zd:",
+  json_element_object_begin (json_ctx);
-	  len, pos, align);
+  json_attr_uint (json_ctx, "align", align);
+  json_attr_uint (json_ctx, "pos", pos);
+  json_attr_uint (json_ctx, "len", len);
+  json_attr_uint (json_ctx, "seek_char", seek_char);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, (CHAR *) (buf + align), seek_char, len);
+    do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len);
 
-  putchar ('\n');
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 }
 
 int
 test_main (void)
 {
   size_t i;
-
+  json_ctx_t json_ctx;
   test_init ();
 
-  printf ("%20s", "");
+  json_init (&json_ctx, 0, stdout);
+
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
+
+  json_array_begin (&json_ctx, "ifuncs");
   FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
+    json_element_string (&json_ctx, impl->name);
-  putchar ('\n');
+  json_array_end (&json_ctx);
+
+  json_array_begin (&json_ctx, "results");
 
   for (i = 1; i < 8; ++i)
     {
-      do_test (0, 16 << i, 2048, 23);
+      do_test (&json_ctx, 0, 16 << i, 2048, 23);
-      do_test (i, 64, 256, 23);
+      do_test (&json_ctx, i, 64, 256, 23);
-      do_test (0, 16 << i, 2048, 0);
+      do_test (&json_ctx, 0, 16 << i, 2048, 0);
-      do_test (i, 64, 256, 0);
+      do_test (&json_ctx, i, 64, 256, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, 64, 256, 0);
 #ifdef USE_AS_MEMRCHR
       /* Also test the position close to the beginning for memrchr.  */
-      do_test (0, i, 256, 23);
+      do_test (&json_ctx, 0, i, 256, 23);
-      do_test (0, i, 256, 0);
+      do_test (&json_ctx, 0, i, 256, 0);
-      do_test (i, i, 256, 23);
+      do_test (&json_ctx, i, i, 256, 23);
-      do_test (i, i, 256, 0);
+      do_test (&json_ctx, i, i, 256, 0);
 #endif
     }
   for (i = 1; i < 8; ++i)
     {
-      do_test (i, i << 5, 192, 23);
+      do_test (&json_ctx, i, i << 5, 192, 23);
-      do_test (i, i << 5, 192, 0);
+      do_test (&json_ctx, i, i << 5, 192, 0);
-      do_test (i, i << 5, 256, 23);
+      do_test (&json_ctx, i, i << 5, 256, 23);
-      do_test (i, i << 5, 256, 0);
+      do_test (&json_ctx, i, i << 5, 256, 0);
-      do_test (i, i << 5, 512, 23);
+      do_test (&json_ctx, i, i << 5, 512, 23);
-      do_test (i, i << 5, 512, 0);
+      do_test (&json_ctx, i, i << 5, 512, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23);
     }
   for (i = 1; i < 32; ++i)
     {
-      do_test (0, i, i + 1, 23);
+      do_test (&json_ctx, 0, i, i + 1, 23);
-      do_test (0, i, i + 1, 0);
+      do_test (&json_ctx, 0, i, i + 1, 0);
-      do_test (i, i, i + 1, 23);
+      do_test (&json_ctx, i, i, i + 1, 23);
-      do_test (i, i, i + 1, 0);
+      do_test (&json_ctx, i, i, i + 1, 0);
-      do_test (0, i, i - 1, 23);
+      do_test (&json_ctx, 0, i, i - 1, 23);
-      do_test (0, i, i - 1, 0);
+      do_test (&json_ctx, 0, i, i - 1, 0);
-      do_test (i, i, i - 1, 23);
+      do_test (&json_ctx, i, i, i - 1, 23);
-      do_test (i, i, i - 1, 0);
+      do_test (&json_ctx, i, i, i - 1, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23);
+      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23);
+      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0);
 #ifdef USE_AS_MEMRCHR
       /* Also test the position close to the beginning for memrchr.  */
-      do_test (0, 1, i + 1, 23);
+      do_test (&json_ctx, 0, 1, i + 1, 23);
-      do_test (0, 2, i + 1, 0);
+      do_test (&json_ctx, 0, 2, i + 1, 0);
 #endif
     }
 
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
+
   return ret;
 }
 

benchtests/bench-memmem.c

@@ -21,6 +21,7 @@
 #define BUF1PAGES 20
 #define ITERATIONS 100
 #include "bench-string.h"
+#include "json-lib.h"
 
 typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
 
@@ -102,44 +103,62 @@ IMPL (twoway_memmem, 0)
 IMPL (basic_memmem, 0)
 
 static void
-do_one_test (impl_t *impl, const void *haystack, size_t haystack_len,
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack,
-	     const void *needle, size_t needle_len, const void *expected)
+	     size_t haystack_len, const void *needle, size_t needle_len,
+	     const void *expected)
 {
   size_t i, iters = INNER_LOOP_ITERS_SMALL;
   timing_t start, stop, cur;
-
+  void *res;
   TIMING_NOW (start);
   for (i = 0; i < iters; ++i)
     {
-      CALL (impl, haystack, haystack_len, needle, needle_len);
+      res = CALL (impl, haystack, haystack_len, needle, needle_len);
     }
   TIMING_NOW (stop);
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
+  json_element_double (json_ctx, (double) cur / (double) iters);
+
+  if (res != expected)
+    {
+      error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)",
+	     res, expected, impl->name, haystack, haystack_len, needle,
+	     needle_len);
+      ret = 1;
+    }
 }
 
 static void
-do_test (const char *str, size_t len, size_t idx)
+do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx)
 {
   char tmpbuf[len];
 
   memcpy (tmpbuf, buf1 + idx, len);
   memcpy (buf1 + idx, str, len);
 
-  printf ("String %s, offset %zd:", str, idx);
+  json_element_object_begin (json_ctx);
+  json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
+  json_attr_uint (json_ctx, "len_needle", len);
+  json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
+  json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str);
+  json_attr_uint (json_ctx, "fail", 0);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx);
+    do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len,
+		 buf1 + idx);
 
   memcpy (buf1 + idx, tmpbuf, len);
 
-  putchar ('\n');
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 }
 
 static void
-do_random_tests (void)
+do_random_tests (json_ctx_t *json_ctx)
 {
   for (size_t n = 0; n < ITERATIONS; ++n)
     {
@@ -159,13 +178,21 @@ do_random_tests (void)
 	  buf1[idx + off] = ch;
 	}
 
-      printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx);
+      json_element_object_begin (json_ctx);
+      json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
+      json_attr_uint (json_ctx, "len_needle", len);
+      json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
+      json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx));
+      json_attr_uint (json_ctx, "fail", 0);
+
+      json_array_begin (json_ctx, "timings");
 
       FOR_EACH_IMPL (impl, 0)
-	do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len,
+	do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx,
-		     buf1 + idx);
+		     len, buf1 + idx);
 
-      putchar ('\n');
+      json_array_end (json_ctx);
+      json_element_object_end (json_ctx);
 
       memcpy (buf1 + idx, tmpbuf, len);
     }
@@ -178,18 +205,28 @@ static const char *const strs[] =
     "abc0", "aaaa0", "abcabc0"
   };
 
-
 int
 test_main (void)
 {
+  json_ctx_t json_ctx;
   size_t i;
 
   test_init ();
+  json_init (&json_ctx, 0, stdout);
 
-  printf ("%23s", "");
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
+
+  json_array_begin (&json_ctx, "ifuncs");
   FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
+    json_element_string (&json_ctx, impl->name);
-  putchar ('\n');
+  json_array_end (&json_ctx);
+
+  json_array_begin (&json_ctx, "results");
 
   for (i = 0; i < BUF1PAGES * page_size; ++i)
     buf1[i] = 60 + random () % 32;
@@ -199,10 +236,15 @@ test_main (void)
       {
 	size_t len = strlen (strs[i]);
 
-	do_test (strs[i], len, j);
+	do_test (&json_ctx, strs[i], len, j);
       }
 
-  do_random_tests ();
+  do_random_tests (&json_ctx);
+
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
   return ret;
 }
 

benchtests/bench-strstr.c

@@ -21,6 +21,8 @@
 #define TEST_NAME "strstr"
 #include "bench-string.h"
 
+#include "json-lib.h"
+
 static const char input[] =
 "This manual is written with the assumption that you are at least "
 "somewhat familiar with the C programming language and basic programming "
@@ -129,7 +131,8 @@ IMPL (twoway_strstr, 0)
 IMPL (basic_strstr, 0)
 
 static void
-do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
+	     const char *s2, char *exp_result)
 {
   size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
   timing_t start, stop, cur;
@@ -142,7 +145,7 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
+  json_element_double (json_ctx, (double) cur / (double) iters);
 
   if (res != exp_result)
     {
@@ -153,10 +156,9 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
     }
 }
 
-
 static void
-do_test (size_t align1, size_t align2, size_t len1, size_t len2,
+do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
-	 int fail)
+	 size_t len2, int fail)
 {
   char *s1 = (char *) (buf1 + align1);
   char *s2 = (char *) (buf2 + align2);
@@ -195,13 +197,21 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
     if (fail || ss1 != s1 + len1 - len2)
       ++ss1[len2 / 2];
 
-  printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:",
+  json_element_object_begin (json_ctx);
-	  len1, len2, align1, align2, fail ? "fail " : "found");
+  json_attr_uint (json_ctx, "len_haystack", len1);
+  json_attr_uint (json_ctx, "len_needle", len2);
+  json_attr_uint (json_ctx, "align_haystack", align1);
+  json_attr_uint (json_ctx, "align_needle", align2);
+  json_attr_uint (json_ctx, "fail", fail);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
+    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
+
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 
-  putchar ('\n');
 }
 
 /* Test needles which exhibit worst-case performance.  This shows that
@@ -211,7 +221,7 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
    within a factor of 2 on several different microarchitectures.  */
 
 static void
-test_hard_needle (size_t ne_len, size_t hs_len)
+test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
 {
   char *ne = (char *) buf1;
   char *hs = (char *) buf2;
@@ -230,11 +240,21 @@ test_hard_needle (size_t ne_len, size_t hs_len)
 	hs[i - 62] = 'b';
       }
 
-    printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len);
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
+
+    json_array_begin (json_ctx, "timings");
 
     FOR_EACH_IMPL (impl, 0)
-      do_one_test (impl, hs, ne, NULL);
+      do_one_test (json_ctx, impl, hs, ne, NULL);
-    putchar ('\n');
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
   }
 
   /* 2nd hard needle for strstr algorithm using skip table.  This results in
@@ -251,11 +271,21 @@ test_hard_needle (size_t ne_len, size_t hs_len)
 	hs[i - 6] = 'b';
       }
 
-    printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len);
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
+
+    json_array_begin (json_ctx, "timings");
 
     FOR_EACH_IMPL (impl, 0)
-      do_one_test (impl, hs, ne, NULL);
+      do_one_test (json_ctx, impl, hs, ne, NULL);
-    putchar ('\n');
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
   }
 
   /* Hard needle for Two-way algorithm - the random input causes a large number
@@ -271,41 +301,124 @@ test_hard_needle (size_t ne_len, size_t hs_len)
     ne[0] = 'b';
     ne[ne_len] = 0;
 
-    printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len);
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult 2-way");
+
+    json_array_begin (json_ctx, "timings");
 
     FOR_EACH_IMPL (impl, 0)
-      do_one_test (impl, hs, ne, NULL);
+      do_one_test (json_ctx, impl, hs, ne, NULL);
-    putchar ('\n');
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
+  }
+
+  /* Hard needle for standard algorithm testing first few characters of
+   * needle.  */
+  {
+    for (int i = 0; i < hs_len; i++)
+      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
+    hs[hs_len] = 0;
+
+    for (int i = 0; i < ne_len; i++)
+      {
+	if (i % 3 == 0)
+	  ne[i] = 'a';
+	else if (i % 3 == 1)
+	  ne[i] = 'b';
+	else
+	  ne[i] = 'c';
+      }
+    ne[ne_len] = 0;
+
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
+
+    json_array_begin (json_ctx, "timings");
+
+    FOR_EACH_IMPL (impl, 0)
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
   }
 }
 
 static int
 test_main (void)
 {
+  json_ctx_t json_ctx;
   test_init ();
 
-  printf ("%23s", "");
+  json_init (&json_ctx, 0, stdout);
-  FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
-  putchar ('\n');
 
-  for (size_t hlen = 64; hlen <= 256; hlen += 32)
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
+
+  json_array_begin (&json_ctx, "ifuncs");
+  FOR_EACH_IMPL (impl, 0)
+    json_element_string (&json_ctx, impl->name);
+  json_array_end (&json_ctx);
+
+  json_array_begin (&json_ctx, "results");
+
+  for (size_t hlen = 8; hlen <= 256;)
     for (size_t klen = 1; klen <= 16; klen++)
       {
-	do_test (1, 3, hlen, klen, 0);
+	do_test (&json_ctx, 1, 3, hlen, klen, 0);
-	do_test (0, 9, hlen, klen, 1);
+	do_test (&json_ctx, 0, 9, hlen, klen, 1);
+
+	do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
+	do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
+
+	do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
+	if (hlen < 64)
+	  {
+	    hlen += 8;
+	  }
+	else
+	  {
+	    hlen += 32;
+	  }
       }
 
   for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
-    for (size_t klen = 16; klen <= 256; klen *= 2)
+    for (size_t klen = 4; klen <= 256; klen *= 2)
       {
-	do_test (1, 11, hlen, klen, 0);
+	do_test (&json_ctx, 1, 11, hlen, klen, 0);
-	do_test (14, 5, hlen, klen, 1);
+	do_test (&json_ctx, 14, 5, hlen, klen, 1);
+
+    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
+    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
+
+	do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
+	do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
+
+	do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
       }
 
-  test_hard_needle (64, 65536);
+  test_hard_needle (&json_ctx, 64, 65536);
-  test_hard_needle (256, 65536);
+  test_hard_needle (&json_ctx, 256, 65536);
-  test_hard_needle (1024, 65536);
+  test_hard_needle (&json_ctx, 1024, 65536);
+
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
 
   return ret;
 }