View Javadoc

1   /**
2    * Copyright (c) 2012, University of Konstanz, Distributed Systems Group
3    * All rights reserved.
4    * 
5    * Redistribution and use in source and binary forms, with or without
6    * modification, are permitted provided that the following conditions are met:
7    * * Redistributions of source code must retain the above copyright
8    * notice, this list of conditions and the following disclaimer.
9    * * Redistributions in binary form must reproduce the above copyright
10   * notice, this list of conditions and the following disclaimer in the
11   * documentation and/or other materials provided with the distribution.
12   * * Neither the name of the University of Konstanz nor the
13   * names of its contributors may be used to endorse or promote products
14   * derived from this software without specific prior written permission.
15   * 
16   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19   * DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
20   * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26   */
27  package org.perfidix;
28  
29  import java.lang.reflect.Method;
30  import java.util.ArrayList;
31  import java.util.HashMap;
32  import java.util.Hashtable;
33  import java.util.LinkedHashSet;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Random;
37  import java.util.Set;
38  
39  import org.perfidix.AbstractConfig.StandardConfig;
40  import org.perfidix.annotation.AfterBenchClass;
41  import org.perfidix.annotation.BeforeBenchClass;
42  import org.perfidix.annotation.Bench;
43  import org.perfidix.element.AbstractMethodArrangement;
44  import org.perfidix.element.BenchmarkElement;
45  import org.perfidix.element.BenchmarkExecutor;
46  import org.perfidix.element.BenchmarkMethod;
47  import org.perfidix.exceptions.PerfidixMethodCheckException;
48  import org.perfidix.exceptions.PerfidixMethodInvocationException;
49  import org.perfidix.meter.AbstractMeter;
50  import org.perfidix.result.BenchmarkResult;
51  
52  /**
53   * Class to hold all classes which want to be benchmarked.
54   * <p>
55   * 
56   * <pre>
57   * 
58   * 
59   * public class MyBenchmark {
60   * 
61   *     public static void main(final String[] args) {
62   *         // Build up own config
63   *         final AbstractConfig config = new MyConfig();
64   * 
65   *         // Initialise benchmark with config
66   *         final Benchmark benchmark = new Benchmark(config);
67   * 
68   *         // Adding Classes to benchmark
69   *         benchmark.add(Class1.class);
70   *         benchmark.add(Class2.class);
71   *         benchmark.add(Object3); // of Class3, every class is allowed to be inserted only once
72   * 
73   *         // Running benchmark
74   *         final BenchmarkResult result = benchmark.run();
75   * 
76   *         // Printing out Result in a Tabular, every AbstractOutput implementing class is allowed.
77   *         new TabularSummaryOutput().visitBenchmark(result);
78   * 
79   *     }
80   * }
81   * 
82   * </pre>
83   * 
84   * </p>
85   * 
86   * @see AbstractConfig
87   * @see BenchmarkResult
88   * @author Sebastian Graf, University of Konstanz
89   */
90  public final class Benchmark {
91  
92      /** Set with all used classes. */
93      private transient final Set<Class<?>> clazzes;
94  
95      /** Already instantiated objects */
96      private transient final Set<Object> objects;
97  
98      /** Simple random for gc-prob */
99      private transient static final Random RAN = new Random();
100 
101     /** Configuration of benchmark, holding everything. */
102     private transient final AbstractConfig conf;
103 
104     /**
105      * Constructor with a fixed set of used meters.
106      * 
107      * @param paramConf
108      *            Configuration for Benchmark
109      */
110     public Benchmark(final AbstractConfig paramConf) {
111         conf = paramConf;
112         this.clazzes = new LinkedHashSet<Class<?>>();
113         this.objects = new LinkedHashSet<Object>();
114 
115     }
116 
117     /**
118      * Convience constructor using the {@link StandardConfig}
119      */
120     public Benchmark() {
121         this(new StandardConfig());
122     }
123 
124     /**
125      * Adding a class to bench to this benchmark. This class should contain
126      * benchmarkable methods, otherwise it will be ignored.
127      * 
128      * @param clazz
129      *            to be added.
130      */
131     public void add(final Class<?> clazz) {
132         if (this.clazzes.contains(clazz)) {
133             throw new IllegalArgumentException("Only one class-instance per benchmark allowed");
134         } else {
135             this.clazzes.add(clazz);
136         }
137     }
138 
139     /**
140      * Adding a already instantiated objects to benchmark. Per benchmark, only
141      * one objects of each class is allowed.
142      * 
143      * @param obj
144      *            to be added
145      */
146     public void add(final Object obj) {
147         final Class<?> clazz = obj.getClass();
148 
149         if (this.clazzes.contains(clazz)) {
150             throw new IllegalArgumentException("Only one class-instance per benchmark allowed");
151         } else {
152             this.clazzes.add(clazz);
153             this.objects.add(obj);
154         }
155     }
156 
157     /**
158      * Getting the number of all methods and all runs
159      * 
160      * @return a map with all methods and the runs.
161      */
162     public Map<BenchmarkMethod, Integer> getNumberOfMethodsAndRuns() {
163         final Map<BenchmarkMethod, Integer> returnVal = new HashMap<BenchmarkMethod, Integer>();
164         final List<BenchmarkMethod> meths = getBenchmarkMethods();
165         for (final BenchmarkMethod meth : meths) {
166             int numberOfRuns = BenchmarkMethod.getNumberOfAnnotatedRuns(meth.getMethodToBench());
167             if (numberOfRuns == Bench.NONE_RUN) {
168                 numberOfRuns = conf.getRuns();
169             }
170             returnVal.put(meth, numberOfRuns);
171         }
172         return returnVal;
173     }
174 
175     /**
176      * Running this benchmark
177      * 
178      * @return {@link BenchmarkResult} the result in an {@link BenchmarkResult} container.
179      */
180     public BenchmarkResult run() {
181         final BenchmarkResult res = new BenchmarkResult(conf.getListener());
182         BenchmarkExecutor.initialize(conf, res);
183 
184         // getting Benchmarkables
185         final List<BenchmarkElement> elements = getBenchmarkElements();
186 
187         // arranging them
188         final AbstractMethodArrangement arrangement =
189             AbstractMethodArrangement.getMethodArrangement(elements, conf.getArrangement());
190 
191         // instantiate methods
192         final Map<Class<?>, Object> instantiatedObj = instantiateMethods(res);
193 
194         // getting the mapping and executing beforemethod
195         final Map<Class<?>, Object> objectsToExecute = executeBeforeBenchClass(instantiatedObj, res);
196 
197         // executing the bench for the arrangement
198         for (final BenchmarkElement elem : arrangement) {
199             // invoking gc if possible
200             if (RAN.nextDouble() < conf.getGcProb()) {
201                 System.gc();
202             }
203 
204             final BenchmarkExecutor exec = BenchmarkExecutor.getExecutor(elem);
205 
206             final Object obj = objectsToExecute.get(elem.getMeth().getMethodToBench().getDeclaringClass());
207             // check needed because of failed initialization of objects
208             if (obj != null) {
209                 exec.executeBeforeMethods(obj);
210                 exec.executeBench(obj);
211                 exec.executeAfterMethods(obj);
212             }
213         }
214 
215         // cleaning up methods to benchmark
216         tearDownObjectsToExecute(objectsToExecute, res);
217         return res;
218     }
219 
220     /**
221      * Setting up executable objects for all registered classes and executing {@link BeforeBenchClass}
222      * annotated methods. If an {@link Exception} occurs, this failure will be stored in the
223      * {@link BenchmarkResult} and
224      * the class will not be instantiated
225      * 
226      * @param res
227      *            {@link BenchmarkResult} for storing possible failures.
228      * @return a mapping with class->objects for all registered classes-
229      */
230     private Map<Class<?>, Object> instantiateMethods(final BenchmarkResult res) {
231         // datastructure initialization for all objects
232         final Map<Class<?>, Object> objectsToUse = new Hashtable<Class<?>, Object>();
233 
234         // generating including already instaniated objects
235         for (final Object obj : this.objects) {
236             final Class<?> clazz = obj.getClass();
237             objectsToUse.put(clazz, obj);
238         }
239 
240         // generating objects for each registered class
241         for (final Class<?> clazz : clazzes) {
242             // generating a new instance on which the benchmark will be
243             // performed if there isn't a user generated one
244             if (!objectsToUse.containsKey(clazz)) {
245                 try {
246                     final Object obj = clazz.newInstance();
247                     objectsToUse.put(clazz, obj);
248                     // otherwise adding an exception to the result
249                 } catch (final InstantiationException e) {
250                     res.addException(new PerfidixMethodInvocationException(e, BeforeBenchClass.class));
251                 } catch (final IllegalAccessException e) {
252                     res.addException(new PerfidixMethodInvocationException(e, BeforeBenchClass.class));
253                 }
254 
255             }
256         }
257 
258         return objectsToUse;
259     }
260 
261     /**
262      * Executing beforeBenchClass if present.
263      * 
264      * @param instantiatedObj
265      *            with the instantiatedObj;
266      * @param res
267      *            where the Exceptions should be stored to
268      * @return valid instances with valid beforeCall
269      */
270     private Map<Class<?>, Object> executeBeforeBenchClass(final Map<Class<?>, Object> instantiatedObj,
271         final BenchmarkResult res) {
272 
273         final Map<Class<?>, Object> returnVal = new Hashtable<Class<?>, Object>();
274 
275         // invoking before bench class
276         for (final Class<?> clazz : instantiatedObj.keySet()) {
277 
278             final Object objectToUse = instantiatedObj.get(clazz);
279             // ..the search for the beforeClassMeth begins...
280             Method beforeClassMeth = null;
281             boolean continueVal = true;
282             try {
283                 beforeClassMeth =
284                     BenchmarkMethod.findAndCheckAnyMethodByAnnotation(clazz, BeforeBenchClass.class);
285                 // ... and if this search is throwing an exception, the
286                 // exception will be added and a flag is set to break up
287             } catch (final PerfidixMethodCheckException e) {
288                 res.addException(e);
289                 continueVal = false;
290             }
291             // if everything worked well...
292             if (continueVal) {
293                 if (beforeClassMeth == null) {
294                     // ...either the objects is directly mapped to the class
295                     // for executing the benches
296                     returnVal.put(clazz, objectToUse);
297                 } else {
298                     // ... or the beforeMethod will be executed and a
299                     // possible exception stored to the result...
300                     final PerfidixMethodCheckException beforeByCheck =
301                         BenchmarkExecutor.checkMethod(objectToUse, BeforeBenchClass.class, beforeClassMeth);
302                     if (beforeByCheck == null) {
303                         final PerfidixMethodInvocationException beforeByInvok =
304                             BenchmarkExecutor.invokeMethod(objectToUse, BeforeBenchClass.class,
305                                 beforeClassMeth);
306                         if (beforeByInvok == null) {
307                             returnVal.put(clazz, objectToUse);
308                         } else {
309                             res.addException(beforeByInvok);
310                         }
311                     } else {
312                         res.addException(beforeByCheck);
313                     }
314                 }
315             }
316         }
317         return returnVal;
318     }
319 
320     /**
321      * Tear down executable objects for all registered classes and executing {@link AfterBenchClass} annotated
322      * methods.
323      * 
324      * @param objects
325      *            a mapping with class->objects to be teared down
326      * @param res
327      *            the {@link BenchmarkResult} for storing possible failures.
328      */
329     private void tearDownObjectsToExecute(final Map<Class<?>, Object> objects, final BenchmarkResult res) {
330 
331         // executing tearDown for all clazzes registered in given Map
332         for (final Class<?> clazz : objects.keySet()) {
333             final Object objectToUse = objects.get(clazz);
334             if (objectToUse != null) {
335                 // executing AfterClass for all objects.
336                 Method afterClassMeth = null;
337                 try {
338                     afterClassMeth =
339                         BenchmarkMethod.findAndCheckAnyMethodByAnnotation(clazz, AfterBenchClass.class);
340                 } catch (final PerfidixMethodCheckException e) {
341                     res.addException(e);
342                 }
343                 // if afterClassMethod exists, the method will be executed and
344                 // possible failures will be stored in the BenchmarkResult
345                 if (afterClassMeth != null) {
346                     final PerfidixMethodCheckException afterByCheck =
347                         BenchmarkExecutor.checkMethod(objectToUse, AfterBenchClass.class, afterClassMeth);
348                     if (afterByCheck == null) {
349                         final PerfidixMethodInvocationException afterByInvok =
350                             BenchmarkExecutor
351                                 .invokeMethod(objectToUse, AfterBenchClass.class, afterClassMeth);
352                         if (afterByInvok != null) {
353                             res.addException(afterByInvok);
354                         }
355                     } else {
356                         res.addException(afterByCheck);
357                     }
358                 }
359             }
360         }
361     }
362 
363     /**
364      * Getting all Benchmarkable methods out of the registered class.
365      * 
366      * @return a Set with {@link BenchmarkMethod}
367      */
368     public List<BenchmarkMethod> getBenchmarkMethods() {
369         // Generating Set for returnVal
370         final List<BenchmarkMethod> elems = new ArrayList<BenchmarkMethod>();
371         // Getting all Methods and testing if its benchmarkable
372         for (final Class<?> clazz : clazzes) {
373             for (final Method meth : clazz.getDeclaredMethods()) {
374                 // Check if benchmarkable, if so, insert to returnVal;
375                 if (BenchmarkMethod.isBenchmarkable(meth)) {
376                     final BenchmarkMethod benchmarkMeth = new BenchmarkMethod(meth);
377                     elems.add(benchmarkMeth);
378                 }
379             }
380         }
381         return elems;
382     }
383 
384     /**
385      * Getting all benchmarkable objects out of the registered classes with the
386      * annotated number of runs.
387      * 
388      * @return a Set with {@link BenchmarkMethod}
389      */
390     public List<BenchmarkElement> getBenchmarkElements() {
391 
392         final List<BenchmarkElement> elems = new ArrayList<BenchmarkElement>();
393 
394         final List<BenchmarkMethod> meths = getBenchmarkMethods();
395 
396         for (final BenchmarkMethod meth : meths) {
397             int numberOfRuns = BenchmarkMethod.getNumberOfAnnotatedRuns(meth.getMethodToBench());
398             if (numberOfRuns == Bench.NONE_RUN) {
399                 numberOfRuns = conf.getRuns();
400             }
401 
402             // getting the number of runs and adding this number of
403             // elements to the set to be evaluated.
404             for (int i = 0; i < numberOfRuns; i++) {
405                 elems.add(new BenchmarkElement(meth));
406             }
407         }
408 
409         return elems;
410     }
411 }