1 /**
2 * Copyright (c) 2012, University of Konstanz, Distributed Systems Group
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the University of Konstanz nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 package org.perfidix;
28
29 import java.lang.reflect.Method;
30 import java.util.ArrayList;
31 import java.util.HashMap;
32 import java.util.Hashtable;
33 import java.util.LinkedHashSet;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.Random;
37 import java.util.Set;
38
39 import org.perfidix.AbstractConfig.StandardConfig;
40 import org.perfidix.annotation.AfterBenchClass;
41 import org.perfidix.annotation.BeforeBenchClass;
42 import org.perfidix.annotation.Bench;
43 import org.perfidix.element.AbstractMethodArrangement;
44 import org.perfidix.element.BenchmarkElement;
45 import org.perfidix.element.BenchmarkExecutor;
46 import org.perfidix.element.BenchmarkMethod;
47 import org.perfidix.exceptions.PerfidixMethodCheckException;
48 import org.perfidix.exceptions.PerfidixMethodInvocationException;
49 import org.perfidix.meter.AbstractMeter;
50 import org.perfidix.result.BenchmarkResult;
51
52 /**
53 * Class to hold all classes which want to be benchmarked.
54 * <p>
55 *
56 * <pre>
57 *
58 *
59 * public class MyBenchmark {
60 *
61 * public static void main(final String[] args) {
62 * // Build up own config
63 * final AbstractConfig config = new MyConfig();
64 *
65 * // Initialise benchmark with config
66 * final Benchmark benchmark = new Benchmark(config);
67 *
68 * // Adding Classes to benchmark
69 * benchmark.add(Class1.class);
70 * benchmark.add(Class2.class);
71 * benchmark.add(Object3); // of Class3, every class is allowed to be inserted only once
72 *
73 * // Running benchmark
74 * final BenchmarkResult result = benchmark.run();
75 *
76 * // Printing out Result in a Tabular, every AbstractOutput implementing class is allowed.
77 * new TabularSummaryOutput().visitBenchmark(result);
78 *
79 * }
80 * }
81 *
82 * </pre>
83 *
84 * </p>
85 *
86 * @see AbstractConfig
87 * @see BenchmarkResult
88 * @author Sebastian Graf, University of Konstanz
89 */
90 public final class Benchmark {
91
92 /** Set with all used classes. */
93 private transient final Set<Class<?>> clazzes;
94
95 /** Already instantiated objects */
96 private transient final Set<Object> objects;
97
98 /** Simple random for gc-prob */
99 private transient static final Random RAN = new Random();
100
101 /** Configuration of benchmark, holding everything. */
102 private transient final AbstractConfig conf;
103
104 /**
105 * Constructor with a fixed set of used meters.
106 *
107 * @param paramConf
108 * Configuration for Benchmark
109 */
110 public Benchmark(final AbstractConfig paramConf) {
111 conf = paramConf;
112 this.clazzes = new LinkedHashSet<Class<?>>();
113 this.objects = new LinkedHashSet<Object>();
114
115 }
116
117 /**
118 * Convience constructor using the {@link StandardConfig}
119 */
120 public Benchmark() {
121 this(new StandardConfig());
122 }
123
124 /**
125 * Adding a class to bench to this benchmark. This class should contain
126 * benchmarkable methods, otherwise it will be ignored.
127 *
128 * @param clazz
129 * to be added.
130 */
131 public void add(final Class<?> clazz) {
132 if (this.clazzes.contains(clazz)) {
133 throw new IllegalArgumentException("Only one class-instance per benchmark allowed");
134 } else {
135 this.clazzes.add(clazz);
136 }
137 }
138
139 /**
140 * Adding a already instantiated objects to benchmark. Per benchmark, only
141 * one objects of each class is allowed.
142 *
143 * @param obj
144 * to be added
145 */
146 public void add(final Object obj) {
147 final Class<?> clazz = obj.getClass();
148
149 if (this.clazzes.contains(clazz)) {
150 throw new IllegalArgumentException("Only one class-instance per benchmark allowed");
151 } else {
152 this.clazzes.add(clazz);
153 this.objects.add(obj);
154 }
155 }
156
157 /**
158 * Getting the number of all methods and all runs
159 *
160 * @return a map with all methods and the runs.
161 */
162 public Map<BenchmarkMethod, Integer> getNumberOfMethodsAndRuns() {
163 final Map<BenchmarkMethod, Integer> returnVal = new HashMap<BenchmarkMethod, Integer>();
164 final List<BenchmarkMethod> meths = getBenchmarkMethods();
165 for (final BenchmarkMethod meth : meths) {
166 int numberOfRuns = BenchmarkMethod.getNumberOfAnnotatedRuns(meth.getMethodToBench());
167 if (numberOfRuns == Bench.NONE_RUN) {
168 numberOfRuns = conf.getRuns();
169 }
170 returnVal.put(meth, numberOfRuns);
171 }
172 return returnVal;
173 }
174
175 /**
176 * Running this benchmark
177 *
178 * @return {@link BenchmarkResult} the result in an {@link BenchmarkResult} container.
179 */
180 public BenchmarkResult run() {
181 final BenchmarkResult res = new BenchmarkResult(conf.getListener());
182 BenchmarkExecutor.initialize(conf, res);
183
184 // getting Benchmarkables
185 final List<BenchmarkElement> elements = getBenchmarkElements();
186
187 // arranging them
188 final AbstractMethodArrangement arrangement =
189 AbstractMethodArrangement.getMethodArrangement(elements, conf.getArrangement());
190
191 // instantiate methods
192 final Map<Class<?>, Object> instantiatedObj = instantiateMethods(res);
193
194 // getting the mapping and executing beforemethod
195 final Map<Class<?>, Object> objectsToExecute = executeBeforeBenchClass(instantiatedObj, res);
196
197 // executing the bench for the arrangement
198 for (final BenchmarkElement elem : arrangement) {
199 // invoking gc if possible
200 if (RAN.nextDouble() < conf.getGcProb()) {
201 System.gc();
202 }
203
204 final BenchmarkExecutor exec = BenchmarkExecutor.getExecutor(elem);
205
206 final Object obj = objectsToExecute.get(elem.getMeth().getMethodToBench().getDeclaringClass());
207 // check needed because of failed initialization of objects
208 if (obj != null) {
209 exec.executeBeforeMethods(obj);
210 exec.executeBench(obj);
211 exec.executeAfterMethods(obj);
212 }
213 }
214
215 // cleaning up methods to benchmark
216 tearDownObjectsToExecute(objectsToExecute, res);
217 return res;
218 }
219
220 /**
221 * Setting up executable objects for all registered classes and executing {@link BeforeBenchClass}
222 * annotated methods. If an {@link Exception} occurs, this failure will be stored in the
223 * {@link BenchmarkResult} and
224 * the class will not be instantiated
225 *
226 * @param res
227 * {@link BenchmarkResult} for storing possible failures.
228 * @return a mapping with class->objects for all registered classes-
229 */
230 private Map<Class<?>, Object> instantiateMethods(final BenchmarkResult res) {
231 // datastructure initialization for all objects
232 final Map<Class<?>, Object> objectsToUse = new Hashtable<Class<?>, Object>();
233
234 // generating including already instaniated objects
235 for (final Object obj : this.objects) {
236 final Class<?> clazz = obj.getClass();
237 objectsToUse.put(clazz, obj);
238 }
239
240 // generating objects for each registered class
241 for (final Class<?> clazz : clazzes) {
242 // generating a new instance on which the benchmark will be
243 // performed if there isn't a user generated one
244 if (!objectsToUse.containsKey(clazz)) {
245 try {
246 final Object obj = clazz.newInstance();
247 objectsToUse.put(clazz, obj);
248 // otherwise adding an exception to the result
249 } catch (final InstantiationException e) {
250 res.addException(new PerfidixMethodInvocationException(e, BeforeBenchClass.class));
251 } catch (final IllegalAccessException e) {
252 res.addException(new PerfidixMethodInvocationException(e, BeforeBenchClass.class));
253 }
254
255 }
256 }
257
258 return objectsToUse;
259 }
260
261 /**
262 * Executing beforeBenchClass if present.
263 *
264 * @param instantiatedObj
265 * with the instantiatedObj;
266 * @param res
267 * where the Exceptions should be stored to
268 * @return valid instances with valid beforeCall
269 */
270 private Map<Class<?>, Object> executeBeforeBenchClass(final Map<Class<?>, Object> instantiatedObj,
271 final BenchmarkResult res) {
272
273 final Map<Class<?>, Object> returnVal = new Hashtable<Class<?>, Object>();
274
275 // invoking before bench class
276 for (final Class<?> clazz : instantiatedObj.keySet()) {
277
278 final Object objectToUse = instantiatedObj.get(clazz);
279 // ..the search for the beforeClassMeth begins...
280 Method beforeClassMeth = null;
281 boolean continueVal = true;
282 try {
283 beforeClassMeth =
284 BenchmarkMethod.findAndCheckAnyMethodByAnnotation(clazz, BeforeBenchClass.class);
285 // ... and if this search is throwing an exception, the
286 // exception will be added and a flag is set to break up
287 } catch (final PerfidixMethodCheckException e) {
288 res.addException(e);
289 continueVal = false;
290 }
291 // if everything worked well...
292 if (continueVal) {
293 if (beforeClassMeth == null) {
294 // ...either the objects is directly mapped to the class
295 // for executing the benches
296 returnVal.put(clazz, objectToUse);
297 } else {
298 // ... or the beforeMethod will be executed and a
299 // possible exception stored to the result...
300 final PerfidixMethodCheckException beforeByCheck =
301 BenchmarkExecutor.checkMethod(objectToUse, BeforeBenchClass.class, beforeClassMeth);
302 if (beforeByCheck == null) {
303 final PerfidixMethodInvocationException beforeByInvok =
304 BenchmarkExecutor.invokeMethod(objectToUse, BeforeBenchClass.class,
305 beforeClassMeth);
306 if (beforeByInvok == null) {
307 returnVal.put(clazz, objectToUse);
308 } else {
309 res.addException(beforeByInvok);
310 }
311 } else {
312 res.addException(beforeByCheck);
313 }
314 }
315 }
316 }
317 return returnVal;
318 }
319
320 /**
321 * Tear down executable objects for all registered classes and executing {@link AfterBenchClass} annotated
322 * methods.
323 *
324 * @param objects
325 * a mapping with class->objects to be teared down
326 * @param res
327 * the {@link BenchmarkResult} for storing possible failures.
328 */
329 private void tearDownObjectsToExecute(final Map<Class<?>, Object> objects, final BenchmarkResult res) {
330
331 // executing tearDown for all clazzes registered in given Map
332 for (final Class<?> clazz : objects.keySet()) {
333 final Object objectToUse = objects.get(clazz);
334 if (objectToUse != null) {
335 // executing AfterClass for all objects.
336 Method afterClassMeth = null;
337 try {
338 afterClassMeth =
339 BenchmarkMethod.findAndCheckAnyMethodByAnnotation(clazz, AfterBenchClass.class);
340 } catch (final PerfidixMethodCheckException e) {
341 res.addException(e);
342 }
343 // if afterClassMethod exists, the method will be executed and
344 // possible failures will be stored in the BenchmarkResult
345 if (afterClassMeth != null) {
346 final PerfidixMethodCheckException afterByCheck =
347 BenchmarkExecutor.checkMethod(objectToUse, AfterBenchClass.class, afterClassMeth);
348 if (afterByCheck == null) {
349 final PerfidixMethodInvocationException afterByInvok =
350 BenchmarkExecutor
351 .invokeMethod(objectToUse, AfterBenchClass.class, afterClassMeth);
352 if (afterByInvok != null) {
353 res.addException(afterByInvok);
354 }
355 } else {
356 res.addException(afterByCheck);
357 }
358 }
359 }
360 }
361 }
362
363 /**
364 * Getting all Benchmarkable methods out of the registered class.
365 *
366 * @return a Set with {@link BenchmarkMethod}
367 */
368 public List<BenchmarkMethod> getBenchmarkMethods() {
369 // Generating Set for returnVal
370 final List<BenchmarkMethod> elems = new ArrayList<BenchmarkMethod>();
371 // Getting all Methods and testing if its benchmarkable
372 for (final Class<?> clazz : clazzes) {
373 for (final Method meth : clazz.getDeclaredMethods()) {
374 // Check if benchmarkable, if so, insert to returnVal;
375 if (BenchmarkMethod.isBenchmarkable(meth)) {
376 final BenchmarkMethod benchmarkMeth = new BenchmarkMethod(meth);
377 elems.add(benchmarkMeth);
378 }
379 }
380 }
381 return elems;
382 }
383
384 /**
385 * Getting all benchmarkable objects out of the registered classes with the
386 * annotated number of runs.
387 *
388 * @return a Set with {@link BenchmarkMethod}
389 */
390 public List<BenchmarkElement> getBenchmarkElements() {
391
392 final List<BenchmarkElement> elems = new ArrayList<BenchmarkElement>();
393
394 final List<BenchmarkMethod> meths = getBenchmarkMethods();
395
396 for (final BenchmarkMethod meth : meths) {
397 int numberOfRuns = BenchmarkMethod.getNumberOfAnnotatedRuns(meth.getMethodToBench());
398 if (numberOfRuns == Bench.NONE_RUN) {
399 numberOfRuns = conf.getRuns();
400 }
401
402 // getting the number of runs and adding this number of
403 // elements to the set to be evaluated.
404 for (int i = 0; i < numberOfRuns; i++) {
405 elems.add(new BenchmarkElement(meth));
406 }
407 }
408
409 return elems;
410 }
411 }