Evaluation: update "real" results with KEDA benchmarks

plots/real-results-static/config-run-analysis.py

@@ -8,7 +8,7 @@ with open('out.json', 'r') as fd:
 
 
 # labels = list(data.keys())
-labels = ['sp-v1', 'sp-v2', '5', '10', '20']
+labels = ['sp-v1', 'sp-v2', 'keda', '5', '10', '20']
 expanded_labels = []
 config_run_times = []
 

plots/real-results-static/out.json

@@ -22,5 +22,10 @@
 "20": [
 {"start_time":"2021-05-09 06:24:15.923901041 +0000 UTC m=+33.337601010","end_time":"2021-05-09 07:17:47.676995401 +0000 UTC m=+3245.090695404","config_runs":50,"mock_settings":"","execution_times":[140.784,120.601,122.578,121.733,125.119,141.477,123.22,122.899,122.253,127.369,143.545,123.169,123.108,123.331,128.208,142.978,121.574,121.638,121.564,126.598,145.224,122.51,122.535,122.568,126.303,142.838,123.258,124.412,123.741,124.12,144.208,127.105,123.959,124.028,129.327,143.507,123.071,125.562,127.22,128.498,147.853,126.951,129.105,126.668,135.173,147.529,126.824,129.641,125.514,126.913,147.488],"time_to_completion":3211.753,"replica_seconds":89540,"extra_applies":"","config_run_failures":0},
 {"start_time":"2021-05-09 07:40:50.507250602 +0000 UTC m=+33.398097721","end_time":"2021-05-09 08:34:19.79667001 +0000 UTC m=+3242.687517161","config_runs":50,"mock_settings":"","execution_times":[141.163,120.384,120.883,121.783,127.588,141.186,121.485,121.894,120.72,130.567,142.918,124.402,123.03,122.539,126.52,142.231,122.557,123.318,122.223,125.951,144.439,121.953,122.615,122.983,125.92,143.45,123.819,125.254,124.336,135.306,144.769,124.484,124.398,124.751,136.389,144.986,125.772,126.079,125.214,128.621,145.583,126.625,125.729,126.691,129.499,146.35,125.416,126.755,126.789,131.18,147.725],"time_to_completion":3209.289,"replica_seconds":89680,"extra_applies":"","config_run_failures":0},
-{"start_time":"2021-05-09 08:57:24.533925363 +0000 UTC m=+33.384380072","end_time":"2021-05-09 09:50:55.912924599 +0000 UTC m=+3244.763379335","config_runs":50,"mock_settings":"","execution_times":[141.13,120.656,121.366,121.632,124.783,140.898,120.334,124.133,120.811,125.309,143.876,123.453,126.827,120.883,125.913,144.111,121.905,123.594,123.364,122.955,142.462,123.393,124.266,122.429,126.604,143.672,123.063,124.473,122.763,130.741,145.318,125.539,125.521,127.408,127.99,147.512,126.016,125.293,126.008,129.316,146.616,128.471,126.702,125.136,127.44,145.688,127.145,126.403,127.622,138.989,148.798],"time_to_completion":3211.378,"replica_seconds":89540,"extra_applies":"","config_run_failures":0}]
+    {"start_time":"2021-05-09 08:57:24.533925363 +0000 UTC m=+33.384380072","end_time":"2021-05-09 09:50:55.912924599 +0000 UTC m=+3244.763379335","config_runs":50,"mock_settings":"","execution_times":[141.13,120.656,121.366,121.632,124.783,140.898,120.334,124.133,120.811,125.309,143.876,123.453,126.827,120.883,125.913,144.111,121.905,123.594,123.364,122.955,142.462,123.393,124.266,122.429,126.604,143.672,123.063,124.473,122.763,130.741,145.318,125.539,125.521,127.408,127.99,147.512,126.016,125.293,126.008,129.316,146.616,128.471,126.702,125.136,127.44,145.688,127.145,126.403,127.622,138.989,148.798],"time_to_completion":3211.378,"replica_seconds":89540,"extra_applies":"","config_run_failures":0}],
+        "keda": [
+        {"start_time":"2021-05-21 05:34:31.648514029 +0000 UTC m=+33.877047750","end_time":"2021-05-21 06:28:05.668965252 +0000 UTC m=+3247.897499003","config_runs":50,"mock_settings":"","execution_times":[445.477,316.829,355.904,364.99,336.497,149.962,284.848,255.046,130.425,206.697,174.348,128.056,131.994,128.188,137.252,148.471,128.827,127.324,128.934,136.755,148.18,129.594,129.511,130.8,134.916,155.345,136.182,133.175,131.128,138.439,149.997,130.736,130.565,130.886,131.879,150.759,130.718,131.272,128.648,133.698,152.65,130.574,130.289,127.714,132.147,151.658,128.988,129.713,131.204,143.46,150.855],"time_to_completion":3214.02,"replica_seconds":36420,"extra_applies":"keda-v1.yaml","config_run_failures":0},
+        {"start_time":"2021-05-21 06:51:06.137035431 +0000 UTC m=+33.790761174","end_time":"2021-05-21 07:44:40.551009428 +0000 UTC m=+3248.204735210","config_runs":50,"mock_settings":"","execution_times":[447.799,314.599,356.213,368.853,335.346,150.153,287.245,253.093,130.453,208.642,174.054,128.176,133.807,126.791,137.143,150.289,130.753,132.841,134.17,132.523,149.253,128.972,126.669,128.423,128.473,152.924,128.031,134.496,130.378,132.558,150.342,134.138,127.636,128.458,131.669,151.874,125.496,130.825,126.471,138.066,148.233,131.406,130.195,128.743,132.26,148.861,129.32,129.107,132.706,144.433,150.885],"time_to_completion":3214.413,"replica_seconds":36485,"extra_applies":"keda-v1.yaml","config_run_failures":0},
+        {"start_time":"2021-05-21 08:07:40.868343719 +0000 UTC m=+33.767244142","end_time":"2021-05-21 09:01:12.045485923 +0000 UTC m=+3244.944386364","config_runs":50,"mock_settings":"","execution_times":[440.643,294.642,334.814,370.537,326.798,150.326,311.833,271.513,151.279,246.524,199.329,150.158,177.706,158.994,155.935,170.895,153.008,146.523,135.714,146.145,150.946,133.693,131.877,131.891,139.5,148.828,131.241,131.14,129.085,142.932,154.073,131.643,135.207,131.091,140.861,151.111,132.384,132.619,134.327,136.092,153.694,136.288,133.714,130.729,139.423,151.374,133.793,132.656,134.64,135.194,148.974],"time_to_completion":3211.177,"replica_seconds":37137,"extra_applies":"keda-v1.yaml","config_run_failures":0}
+    ]
 }

plots/real-results-static/performance-analysis.py

@@ -8,7 +8,7 @@ with open('out.json', 'r') as fd:
 
 
 # labels = list(data.keys())
-labels = ['sp-v1', 'sp-v2', '5', '10', '20']
+labels = ['sp-v1', 'sp-v2', 'keda', '5', '10', '20']
 
 fig, axr = plt.subplots(nrows=1, ncols=1, figsize=(7, 3))
 axr.grid(True, zorder=1.0)

plots/real-results-variable/config-run-analysis.py

@@ -8,7 +8,7 @@ with open('out.json', 'r') as fd:
 
 
 #labels = list(data.keys())
-labels = ["sp-v1", "sp-v2", "5", "10", "20"]
+labels = ["sp-v1", "sp-v2", "keda", "5", "10", "20"]
 expanded_labels = []
 config_run_times = []
 

plots/real-results-variable/out.json

@@ -34,5 +34,10 @@
 "20": [
 {"start_time":"2021-05-07 00:02:49.562760378 +0000 UTC m=+33.326969709","end_time":"2021-05-07 00:56:16.335759098 +0000 UTC m=+3240.099968457","config_runs":50,"mock_settings":"","execution_times":[141.235,120.685,123.421,124.071,133.995,143.345,123.505,123.386,142.693,121.807,122.166,122.404,124.158,144.624,121.972,122.037,141.174,123.57,125.12,123.561,126.456,143.973,122.765,123.222,143.25,124.214,125.493,123.433,128.366,144.347,125.142,125.283,143.156,125.434,125.679,123.339,128.978,146.215,125.516,126.355,145.743],"time_to_completion":3206.772,"replica_seconds":83680,"extra_applies":"","config_run_failures":0},
 {"start_time":"2021-05-08 00:27:51.239612681 +0000 UTC m=+33.339804984","end_time":"2021-05-08 01:21:17.949784918 +0000 UTC m=+3240.049977241","config_runs":50,"mock_settings":"","execution_times":[142.996,122.233,122.777,121.933,124.401,142.403,122.904,123.733,141.369,122.295,125.021,124.03,125.601,143.979,123.859,123.79,142.723,125.058,124.654,125.112,130.075,144.611,126.147,126.6,141.028,127.081,125.577,124.554,130.46,144.047,124.078,126.091,144.164,125.548,125.474,123.819,125.893,145.645,124.551,124.078,143.987],"time_to_completion":3206.71,"replica_seconds":83740,"extra_applies":"","config_run_failures":0},
-{"start_time":"2021-05-08 01:39:25.210643473 +0000 UTC m=+33.355658581","end_time":"2021-05-08 02:32:51.777616682 +0000 UTC m=+3239.922631800","config_runs":50,"mock_settings":"","execution_times":[140.505,118.931,120.434,121.062,124.533,141.928,121.173,121.802,141.029,124.014,122.56,120.892,126.68,143.255,121.146,125.34,142.872,123.523,123.068,124.728,126.437,145.333,124.661,124.285,142.565,125.275,124.698,123.269,131.569,145.803,124.526,124.525,144.179,125.082,125.684,124.387,134.878,145.942,126.865,126.152,143.985],"time_to_completion":3206.566,"replica_seconds":83620,"extra_applies":"","config_run_failures":0}]
+    {"start_time":"2021-05-08 01:39:25.210643473 +0000 UTC m=+33.355658581","end_time":"2021-05-08 02:32:51.777616682 +0000 UTC m=+3239.922631800","config_runs":50,"mock_settings":"","execution_times":[140.505,118.931,120.434,121.062,124.533,141.928,121.173,121.802,141.029,124.014,122.56,120.892,126.68,143.255,121.146,125.34,142.872,123.523,123.068,124.728,126.437,145.333,124.661,124.285,142.565,125.275,124.698,123.269,131.569,145.803,124.526,124.525,144.179,125.082,125.684,124.387,134.878,145.942,126.865,126.152,143.985],"time_to_completion":3206.566,"replica_seconds":83620,"extra_applies":"","config_run_failures":0}],
+    "keda": [
+        {"start_time":"2021-05-21 09:24:15.465630929 +0000 UTC m=+33.781562337","end_time":"2021-05-21 10:17:48.064192206 +0000 UTC m=+3246.380123646","config_runs":50,"mock_settings":"","execution_times":[448.025,314.342,357.354,367.111,335.457,148.935,287.714,255.904,146.954,130.381,128.336,127.69,129.116,148.613,129.87,131.441,452.974,216.563,258.071,371.12,237.625,152.328,291.561,155.339,147.116,129.108,129.06,129.527,138.958,149.149,128.938,129.484,452.38,217.695,257.746,371.843,236.955,153.125,289.739,155.206,147.762],"time_to_completion":3212.598,"replica_seconds":37280,"extra_applies":"keda-v1.yaml","config_run_failures":0},
+        {"start_time":"2021-05-21 10:35:50.112553505 +0000 UTC m=+33.713988980","end_time":"2021-05-21 11:29:21.289689196 +0000 UTC m=+3244.891124706","config_runs":50,"mock_settings":"","execution_times":[430.277,299.398,339.134,351.652,317.673,150.199,271.709,237.759,149.09,128.632,130.029,125.655,130.783,150.458,129.605,129.35,451.717,214.783,253.645,369.533,233.397,151.892,288.526,152.08,146.404,126.201,129.991,127.575,134.991,149.187,132.069,132.565,464.58,231.326,273.222,177.668,325.111,155.888,302.692,170.017,148.342],"time_to_completion":3211.177,"replica_seconds":39071,"extra_applies":"keda-v1.yaml","config_run_failures":0},
+        {"start_time":"2021-05-21 11:47:24.639495997 +0000 UTC m=+33.747255758","end_time":"2021-05-21 12:40:57.173447232 +0000 UTC m=+3246.281207037","config_runs":50,"mock_settings":"","execution_times":[449.395,312.66,351.91,368.903,331.996,148.758,287.27,250.979,147.761,127.258,131.37,128.405,132.701,148.453,128.553,128.646,453.269,212.718,252.146,372.411,232.066,151.404,291.317,151.31,146.598,128.061,130.864,126.84,138.026,150.411,131.388,132.898,317.808,240.485,279.657,197.696,259.782,165.974,144.05,178.413,147.232],"time_to_completion":3212.533,"replica_seconds":38995,"extra_applies":"keda-v1.yaml","config_run_failures":0}
+    ]
 }

plots/real-results-variable/performance-analysis.py

@@ -7,10 +7,12 @@ with open('out.json', 'r') as fd:
     data = json.load(fd)
 
 
-labels = list(data.keys())
-labels = ["sp-v1", "sp-v2", "5", "10", "20"]
+# labels = list(data.keys())
+#labels = ['2', '3', '5', '10', '15', '20', 'hpav1']
+#labels = ['5', '10', 'hpav1', 'hpav2', 'hpav3', 'hpav4', 'hpav5']
+labels = ['sp-v1', 'sp-v2', 'keda', '5', '10', '20']
 
-fig, axr = plt.subplots(nrows=1, ncols=1, figsize=(7, 3))
+fig, axr = plt.subplots(nrows=1, ncols=1, figsize=(7, 3.5))
 axr.grid(True, zorder=1.0)
 
 for label in labels: