View Javadoc
1   /*
2    * Copyright (C) 2009 Jayway AB
3    * Copyright (C) 2007-2008 JVending Masa
4    *
5    * Licensed under the Apache License, Version 2.0 (the "License");
6    * you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package com.simpligility.maven.plugins.android.standalonemojos;
18  
19  import com.android.ddmlib.IDevice;
20  import com.android.ddmlib.testrunner.ITestRunListener;
21  import com.android.ddmlib.testrunner.TestIdentifier;
22  import com.simpligility.maven.plugins.android.AbstractAndroidMojo;
23  import com.simpligility.maven.plugins.android.AndroidTestRunListener;
24  import com.simpligility.maven.plugins.android.CommandExecutor;
25  import com.simpligility.maven.plugins.android.DeviceCallback;
26  import com.simpligility.maven.plugins.android.ExecutionException;
27  import com.simpligility.maven.plugins.android.config.ConfigHandler;
28  import com.simpligility.maven.plugins.android.config.ConfigPojo;
29  import com.simpligility.maven.plugins.android.config.PullParameter;
30  import com.simpligility.maven.plugins.android.configuration.MonkeyRunner;
31  import com.simpligility.maven.plugins.android.configuration.Program;
32  
33  import org.apache.commons.lang3.StringUtils;
34  import org.apache.maven.plugin.MojoExecutionException;
35  import org.apache.maven.plugin.MojoFailureException;
36  import org.apache.maven.plugins.annotations.Mojo;
37  import org.apache.maven.plugins.annotations.Parameter;
38  import org.codehaus.plexus.interpolation.os.Os;
39  import org.codehaus.plexus.util.cli.shell.BourneShell;
40  
41  import java.io.File;
42  import java.util.ArrayList;
43  import java.util.HashMap;
44  import java.util.List;
45  import java.util.Map;
46  import java.util.regex.Matcher;
47  import java.util.regex.Pattern;
48  
49  /**
50   * Can execute monkey runner programs.<br>
51   * Implements parsing parameters from pom or command line arguments and sets useful defaults as well. This goal will
52   * invoke monkey runner scripts. If the application crashes during the exercise, this goal can fail the build. <br>
53   * A typical usage of this goal can be found at <a
54   * href="https://github.com/stephanenicolas/Quality-Tools-for-Android">Quality tools for Android project</a>.
55   * 
56   * @see <a href="http://developer.android.com/tools/help/monkey.html">Monkey docs by Google</a>
57   * @see <a href="http://stackoverflow.com/q/3968064/693752">Stack Over Flow thread for parsing monkey output.</a>
58   * @author Stéphane Nicolas - snicolas@octo.com
59   */
60  @SuppressWarnings( "unused" )
61  @Mojo( name = "monkeyrunner" )
62  public class MonkeyRunnerMojo extends AbstractAndroidMojo
63  {
64      /**
65       * -Dmaven.test.skip is commonly used with Maven to skip tests. We honor it.
66       */
67      @Parameter( property = "maven.test.skip", defaultValue = "false", readonly = true )
68      private boolean mavenTestSkip;
69  
70      /**
71       * -DskipTests is commonly used with Maven to skip tests. We honor it too.
72       */
73      @Parameter( property = "skipTests", defaultValue = "false", readonly = true )
74      private boolean mavenSkipTests;
75      /**
76       * -Dmaven.test.failure.ignore is commonly used with Maven to prevent failure of build when (some) tests fail. We
77       * honor it too.
78       */
79      @Parameter( property = "maven.test.failure.ignore", defaultValue = "false", readonly = true )
80      private boolean mavenTestFailureIgnore;
81  
82      /**
83       * -Dmaven.test.failure.ignore is commonly used with Maven to prevent failure of build when (some) tests fail. We
84       * honor it too.
85       */
86      @Parameter( property = "testFailureIgnore", defaultValue = "false", readonly = true )
87      private boolean mavenIgnoreTestFailure;
88  
89      /**
90       * The configuration for the monkey runner goal.
91       * 
92       * <pre>
93       * &lt;monkeyrunner&gt;
94       *   &lt;skip&gt;false&lt;/skip&gt;
95       * &lt;/monkeyrunner&gt;
96       * </pre>
97       * 
98       * Full configuration can use these parameters.
99       * 
100      * <pre>
101      *  &lt;monkeyrunner&gt;
102      *    &lt;skip&gt;false&lt;/skip&gt;
103      *    &lt;createReport&gt;true&lt;/createReport&gt;
104      *  &lt;/monkeyrunner&gt;
105      * </pre>
106      * 
107      * Alternatively to the plugin configuration values can also be configured as properties on the command line as
108      * android.lint.* or in pom or settings file as properties like lint*.
109      */
110     @Parameter
111     @ConfigPojo
112     private MonkeyRunner monkeyrunner;
113 
114     /**
115      * Enables or disables monkey runner test goal. If <code>true</code> it will be skipped; if <code>false</code>, it
116      * will be run. Defaults to true.
117      */
118     @Parameter( property = "android.monkeyrunner.skip" )
119     private Boolean monkeyRunnerSkip;
120 
121     @PullParameter( defaultValue = "true" )
122     private Boolean parsedSkip;
123 
124     /**
125      * (Optional) Specifies a .jar file containing a plugin for monkeyrunner. To learn more about monkeyrunner plugins,
126      * see <a href="http://developer.android.com/tools/help/monkeyrunner_concepts.html#Plugins">Extending monkeyrunner
127      * with plugins</a>. You can add as many plugins as you want.
128      * 
129      * Defaults to no plugins.
130      */
131     @Parameter( property = "android.monkeyrunner.plugins" )
132     private String[] monkeyPlugins;
133 
134     @PullParameter( defaultValueGetterMethod = "getPlugins" )
135     private String[] parsedPlugins;
136 
137     /**
138      * Runs the contents of the file as a Python program.
139      * 
140      * <pre>
141      * &lt;programs&gt;
142      *   &lt;program&gt;
143      *     &lt;filename&gt;foo.py&lt;/filename&gt;
144      *     &lt;options&gt;bar&lt;/options&gt;
145      *   &lt;program&gt;
146      *   &lt;program&gt;
147      *     &lt;filename&gt;foo2.py&lt;/filename&gt;
148      *   &lt;program&gt;
149      *   [..]
150      * &lt;/programs&gt;
151      * </pre>
152      */
153     @Parameter
154     @PullParameter( required = false, defaultValueGetterMethod = "getPrograms" )
155     private List< Program > parsedPrograms;
156 
157     /**
158      * Create a junit xml format compatible output file containing the test results for each device the instrumentation
159      * tests run on. <br>
160      * <br>
161      * The files are stored in target/surefire-reports and named TEST-deviceid.xml. The deviceid for an emulator is
162      * deviceSerialNumber_avdName_manufacturer_model. The serial number is commonly emulator-5554 for the first emulator
163      * started with numbers increasing. avdName is as defined in the SDK tool. The manufacturer is typically "unknown"
164      * and the model is typically "sdk".<br>
165      * The deviceid for an actual devices is deviceSerialNumber_manufacturer_model. <br>
166      * <br>
167      * The file contains system properties from the system running the Android Maven Plugin (JVM) and device properties
168      * from the device/emulator the tests are running on. <br>
169      * <br>
170      * The file contains a single TestSuite for all tests and a TestCase for each test method. Errors and failures are
171      * logged in the file and the system log with full stack traces and other details available.
172      * 
173      * Defaults to false.
174      *
175      */
176     @Parameter( property = "android.monkeyrunner.createReport" )
177     private Boolean monkeyCreateReport;
178 
179     @PullParameter( defaultValue = "false" )
180     private Boolean parsedCreateReport;
181 
182     /**
183      * Decides whether or not to inject device serial number as a parameter to each monkey runner script. The parameter
184      * will be the first parameter passed to the script. This parameter allows to support monkey runner tests on
185      * multiple devices. In that case, monkey runner scripts have to be modified to take the new parameter into account.
186      * Follow that <a href="http://stackoverflow.com/a/13460438/693752">thread on stack over flow to learn more about
187      * it</a>.
188      */
189     @Parameter( property = "android.monkeyrunner.injectDeviceSerialNumberIntoScript" )
190     private Boolean monkeyInjectDeviceSerialNumberIntoScript;
191 
192     @PullParameter( defaultValue = "false" )
193     private Boolean parsedInjectDeviceSerialNumberIntoScript;
194 
195     private long elapsedTime;
196 
197     private ITestRunListener[] mTestListeners;
198 
199     private Map< String, String > runMetrics;
200 
201     private String mRunName;
202 
203     private int eventCount;
204 
205     private TestIdentifier mCurrentTestIndentifier;
206 
207     private MonkeyRunnerErrorListener errorListener;
208 
209     @Override
210     public void execute() throws MojoExecutionException, MojoFailureException
211     {
212         ConfigHandler configHandler = new ConfigHandler( this, this.session, this.execution );
213         configHandler.parseConfiguration();
214 
215         doWithDevices( new DeviceCallback()
216         {
217             @Override
218             public void doWithDevice( IDevice device ) throws MojoExecutionException, MojoFailureException
219             {
220                 AndroidTestRunListener testRunListener = new AndroidTestRunListener( device, getLog(),
221                         parsedCreateReport, false, "", "", targetDirectory );
222                 if ( isEnableIntegrationTest() )
223                 {
224                     run( device, testRunListener );
225                 }
226             }
227         } );
228     }
229 
230     /**
231      * Whether or not tests are enabled.
232      * 
233      * @return a boolean indicating whether or not tests are enabled.
234      */
235     protected boolean isEnableIntegrationTest()
236     {
237         return !parsedSkip && !mavenTestSkip && !mavenSkipTests;
238     }
239 
240     /**
241      * Whether or not test failures should be ignored.
242      * 
243      * @return a boolean indicating whether or not test failures should be ignored.
244      */
245     protected boolean isIgnoreTestFailures()
246     {
247         return mavenIgnoreTestFailure || mavenTestFailureIgnore;
248     }
249 
250     /**
251      * Actually plays tests.
252      * 
253      * @param device
254      *            the device on which tests are going to be executed.
255      * @param iTestRunListeners
256      *            test run listeners.
257      * @throws MojoExecutionException
258      *             if exercising app threw an exception and isIgnoreTestFailures is false..
259      * @throws MojoFailureException
260      *             if exercising app failed and isIgnoreTestFailures is false.
261      */
262     protected void run( IDevice device, ITestRunListener... iTestRunListeners ) throws MojoExecutionException,
263             MojoFailureException
264     {
265 
266         this.mTestListeners = iTestRunListeners;
267 
268         getLog().debug( "Parsed values for Android Monkey Runner invocation: " );
269 
270         CommandExecutor executor = CommandExecutor.Factory.createDefaultCommmandExecutor();
271         if ( !Os.isFamily( Os.FAMILY_WINDOWS ) )
272         {
273             executor.setCustomShell( new CustomBourneShell() );
274         }
275         executor.setLogger( this.getLog() );
276 
277         String command = getAndroidSdk().getMonkeyRunnerPath();
278 
279         List< String > pluginParameters = new ArrayList< String >();
280 
281         if ( parsedPlugins != null && parsedPlugins.length != 0 )
282         {
283             for ( String plugin : parsedPlugins )
284             {
285                 String pluginFilePath = new File( project.getBasedir(), plugin ).getAbsolutePath();
286                 pluginParameters.add( "-plugin " + pluginFilePath );
287             }
288         }
289 
290         if ( parsedPrograms != null && !parsedPrograms.isEmpty() )
291         {
292             handleTestRunStarted();
293             errorListener = new MonkeyRunnerErrorListener();
294             executor.setErrorListener( errorListener );
295 
296             for ( Program program : parsedPrograms )
297             {
298                 List< String > parameters = new ArrayList< String >( pluginParameters );
299 
300                 String programFileName = new File( project.getBasedir(), program.getFilename() ).getAbsolutePath();
301                 parameters.add( programFileName );
302                 String testName = programFileName;
303                 if ( testName.contains( "/" ) )
304                 {
305                     testName.substring( testName.indexOf( '/' ) + 1 );
306                 }
307                 mCurrentTestIndentifier = new TestIdentifier( "MonkeyTest ", testName );
308 
309                 String programOptions = program.getOptions();
310                 if ( parsedInjectDeviceSerialNumberIntoScript != null && parsedInjectDeviceSerialNumberIntoScript )
311                 {
312                     parameters.add( device.getSerialNumber() );
313                 }
314                 if ( programOptions != null && !StringUtils.isEmpty( programOptions ) )
315                 {
316                     parameters.add( programOptions );
317                 }
318 
319                 try
320                 {
321                     getLog().info( "Running command: " + command );
322                     getLog().info( "with parameters: " + parameters );
323                     handleTestStarted();
324                     executor.setCaptureStdOut( true );
325                     executor.executeCommand( command, parameters, true );
326                     handleTestEnded();
327                 }
328                 catch ( ExecutionException e )
329                 {
330                     getLog().info( "Monkey runner produced errors" );
331                     handleTestRunFailed( e.getMessage() );
332 
333                     if ( !isIgnoreTestFailures() )
334                     {
335                         getLog().info( "Project is configured to fail on error." );
336                         getLog().info(
337                                 "Inspect monkey runner reports or re-run with -X to see monkey runner errors in log" );
338                         getLog().info( "Failing build as configured. Ignore following error message." );
339                         if ( errorListener.hasError )
340                         {
341                             getLog().info( "Stack trace is:" );
342                             getLog().info( errorListener.getStackTrace() );
343                         }
344                         throw new MojoExecutionException( "", e );
345                     }
346                 }
347 
348                 if ( errorListener.hasError() )
349                 {
350                     handleCrash();
351                 }
352             }
353             handleTestRunEnded();
354         }
355 
356         getLog().info( "Monkey runner test runs completed successfully." );
357     }
358 
359     private void handleTestRunStarted()
360     {
361         runMetrics = new HashMap< String, String >();
362         elapsedTime = System.currentTimeMillis();
363         for ( ITestRunListener listener : mTestListeners )
364         {
365             listener.testRunStarted( mRunName, eventCount );
366         }
367     }
368 
369     private void handleTestRunFailed( String error )
370     {
371         for ( ITestRunListener listener : mTestListeners )
372         {
373             listener.testRunFailed( error );
374         }
375     }
376 
377     private void handleTestRunEnded()
378     {
379         elapsedTime = System.currentTimeMillis() - elapsedTime;
380 
381         for ( ITestRunListener listener : mTestListeners )
382         {
383             listener.testRunEnded( elapsedTime, runMetrics );
384         }
385     }
386 
387     private void handleTestStarted()
388     {
389         System.out.println( "TEST START " + mTestListeners.length );
390         for ( ITestRunListener listener : mTestListeners )
391         {
392             listener.testStarted( mCurrentTestIndentifier );
393         }
394     }
395 
396     private void handleTestEnded()
397     {
398         if ( mCurrentTestIndentifier != null )
399         {
400             for ( ITestRunListener listener : mTestListeners )
401             {
402                 listener.testEnded( mCurrentTestIndentifier, new HashMap< String, String >() );
403             }
404             mCurrentTestIndentifier = null;
405         }
406     }
407 
408     private void handleCrash()
409     {
410 
411         String trace = errorListener.getStackTrace();
412 
413         for ( ITestRunListener listener : mTestListeners )
414         {
415             listener.testFailed( mCurrentTestIndentifier, trace );
416         }
417         mCurrentTestIndentifier = null;
418 
419     }
420 
421     private final class MonkeyRunnerErrorListener implements CommandExecutor.ErrorListener
422     {
423         private StringBuilder stackTraceBuilder = new StringBuilder();
424         private boolean hasError = false;
425 
426         @Override
427         public boolean isError( String error )
428         {
429 
430             // Unconditionally ignore *All* build warning if configured to
431             if ( isIgnoreTestFailures() )
432             {
433                 return false;
434             }
435 
436             if ( hasError )
437             {
438                 stackTraceBuilder.append( error ).append( '\n' );
439             }
440 
441             final Pattern pattern = Pattern.compile( ".*error.*|.*exception.*", Pattern.CASE_INSENSITIVE );
442             final Matcher matcher = pattern.matcher( error );
443 
444             // If the the reg.exp actually matches, we can safely say this is not an error
445             // since in theory the user told us so
446             if ( matcher.matches() )
447             {
448                 hasError = true;
449                 stackTraceBuilder.append( error ).append( '\n' );
450                 return true;
451             }
452 
453             // Otherwise, it is just another error
454             return false;
455         }
456 
457         public String getStackTrace()
458         {
459             if ( hasError )
460             {
461                 return stackTraceBuilder.toString();
462             }
463             else
464             {
465                 return null;
466             }
467         }
468 
469         public boolean hasError()
470         {
471             return hasError;
472         }
473     }
474 
475     /**
476      * @return default plugins.
477      */
478     public String[] getPlugins()
479     {
480         return parsedPlugins;
481     }
482 
483     private static final class CustomBourneShell extends BourneShell
484     {
485         @Override
486         public List< String > getShellArgsList()
487         {
488             List< String > shellArgs = new ArrayList< String >();
489             List< String > existingShellArgs = super.getShellArgsList();
490 
491             if ( existingShellArgs != null && !existingShellArgs.isEmpty() )
492             {
493                 shellArgs.addAll( existingShellArgs );
494             }
495 
496             return shellArgs;
497         }
498 
499         @Override
500         public String[] getShellArgs()
501         {
502             String[] shellArgs = super.getShellArgs();
503             if ( shellArgs == null )
504             {
505                 shellArgs = new String[ 0 ];
506             }
507 
508             return shellArgs;
509         }
510 
511     }
512 
513     public List< Program > getPrograms()
514     {
515         // return null if not set
516         return parsedPrograms;
517     }
518 }