View Javadoc
1   /*
2    * Copyright (C) 2009-2011 Jayway AB
3    * Copyright (C) 2007-2008 JVending Masa
4    *
5    * Licensed under the Apache License, Version 2.0 (the "License");
6    * you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package com.simpligility.maven.plugins.android;
18  
19  import com.android.ddmlib.AdbCommandRejectedException;
20  import com.android.ddmlib.IDevice;
21  import com.android.ddmlib.ShellCommandUnresponsiveException;
22  import com.android.ddmlib.TimeoutException;
23  import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
24  import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
25  import com.simpligility.maven.plugins.android.asm.AndroidTestFinder;
26  import com.simpligility.maven.plugins.android.common.DeviceHelper;
27  import com.simpligility.maven.plugins.android.configuration.Test;
28  
29  import org.apache.commons.lang3.StringUtils;
30  import org.apache.maven.plugin.MojoExecutionException;
31  import org.apache.maven.plugin.MojoFailureException;
32  import org.apache.maven.plugins.annotations.Parameter;
33  
34  import java.io.IOException;
35  import java.util.ArrayList;
36  import java.util.List;
37  import java.util.Map;
38  
39  /**
40   * AbstractInstrumentationMojo implements running the instrumentation
41   * tests.
42   *
43   * @author hugo.josefson@jayway.com
44   * @author Manfred Moser - manfred@simpligility.com
45   */
46  public abstract class AbstractInstrumentationMojo extends AbstractAndroidMojo
47  {
48  
49      /**
50       * -Dmaven.test.skip is commonly used with Maven to skip tests. We honor it too.
51       */
52      @Parameter( property = "maven.test.skip", defaultValue = "false", readonly = true )
53      private boolean mavenTestSkip;
54  
55      /**
56       * -DskipTests is commonly used with Maven to skip tests. We honor it too.
57       */
58      @Parameter( property = "skipTests", defaultValue = "false", readonly = true )
59      private boolean mavenSkipTests;
60  
61      /**
62       * -Dmaven.test.failure.ignore is commonly used with Maven to ignore test failures. We honor it too.
63       * Ignore or not tests failures. If <code>true</code> they will be ignored; if
64       * <code>false</code>, they will not. Default value is <code>false</code>.
65       */
66      @Parameter( property = "maven.test.failure.ignore", defaultValue = "false", readonly = true )
67      private boolean mavenIgnoreTestFailure;
68  
69      /**
70       * -Dmaven.test.error.ignore is commonly used with Maven to ignore test errors. We honor it too.
71       * Ignore or not tests errors. If <code>true</code> they will be ignored; if
72       * <code>false</code>, they will not. Default value is <code>false</code>.
73       */
74      @Parameter( property = "maven.test.error.ignore", defaultValue = "false", readonly = true )
75      private boolean mavenIgnoreTestError;
76  
77      /**
78       * The configuration to use for running instrumentation tests. Complete configuration
79       * is possible in the plugin configuration:
80       * <pre>
81       * &lt;test&gt;
82       *   &lt;skip&gt;true|false|auto&lt;/skip&gt;
83       *   &lt;instrumentationPackage&gt;packageName&lt;/instrumentationPackage&gt;
84       *   &lt;instrumentationRunner&gt;className&lt;/instrumentationRunner&gt;
85       *   &lt;debug&gt;true|false&lt;/debug&gt;
86       *   &lt;coverage&gt;true|false&lt;/coverage&gt;
87       *   &lt;coverageFile&gt;&lt;/coverageFile&gt;
88       *   &lt;logOnly&gt;true|false&lt;/logOnly&gt;  avd
89       *   &lt;testSize&gt;small|medium|large&lt;/testSize&gt;
90       *   &lt;createReport&gt;true|false&lt;/createReport&gt;
91       *   &lt;classes&gt;
92       *     &lt;class&gt;your.package.name.YourTestClass&lt;/class&gt;
93       *   &lt;/classes&gt;
94       *   &lt;packages&gt;
95       *     &lt;package&gt;your.package.name&lt;/package&gt;
96       *   &lt;/packages&gt;
97       *   &lt;instrumentationArgs&gt;
98       *     &lt;instrumentationArg&gt;key value&lt;/instrumentationArg&gt;
99       *   &lt;/instrumentationArgs&gt;
100      * &lt;/test&gt;
101      * </pre>
102      */
103     @Parameter
104     private Test test;
105 
106     /**
107      * Enables or disables integration test related goals. If <code>true</code> they will be skipped; if
108      * <code>false</code>, they will be run. If <code>auto</code>, they will run if any of the classes inherit from any
109      * class in <code>junit.framework.**</code> or <code>android.test.**</code>.
110      */
111     @Parameter( property = "android.test.skip", defaultValue = "auto" )
112     private String testSkip;
113 
114     /**
115      * Enables or disables integration safe failure.
116      * If <code>true</code> build will not stop on test failure or error.
117      */
118     @Parameter( property = "android.test.failsafe", defaultValue = "true" )
119     private Boolean testFailSafe;
120 
121     /**
122      * Package name of the apk we wish to instrument. If not specified, it is inferred from
123      * <code>AndroidManifest.xml</code>.
124      */
125     @Parameter( property = "android.test.instrumentationPackage" )
126     private String testInstrumentationPackage;
127 
128     /**
129      * Class name of test runner. If not specified, it is inferred from <code>AndroidManifest.xml</code>.
130      */
131     @Parameter( property = "android.test.instrumentationRunner" )
132     private String testInstrumentationRunner;
133 
134     /**
135      * Enable debug causing the test runner to wait until debugger is
136      * connected with the Android debug bridge (adb).
137      */
138     @Parameter( property = "android.test.debug", defaultValue = "false" )
139     private Boolean testDebug;
140 
141     /**
142      * Enable or disable code coverage for this instrumentation test run.
143      */
144     @Parameter( property = "android.test.coverage", defaultValue = "false" )
145     private Boolean testCoverage;
146 
147     /**
148      * Location on device into which coverage should be stored (blank for
149      * Android default /data/data/your.package.here/files/coverage.ec).
150      */
151     @Parameter( property = "android.test.coverageFile" )
152     private String testCoverageFile;
153 
154     /**
155      * Enable this flag to run a log only and not execute the tests.
156      */
157     @Parameter( property = "android.test.logonly", defaultValue = "false" )
158     private Boolean testLogOnly;
159 
160     /**
161      * If specified only execute tests of certain size as defined by
162      * the Android instrumentation testing SmallTest, MediumTest and
163      * LargeTest annotations. Use "small", "medium" or "large" as values.
164      *
165      * @see com.android.ddmlib.testrunner.IRemoteAndroidTestRunner
166      */
167     @Parameter( property = "android.test.testsize" )
168     private String testTestSize;
169 
170     /**
171      * Create a junit xml format compatible output file containing
172      * the test results for each device the instrumentation tests run
173      * on.
174      * <br><br>
175      * The files are stored in target/surefire-reports and named TEST-deviceid.xml.
176      * The deviceid for an emulator is deviceSerialNumber_avdName_manufacturer_model.
177      * The serial number is commonly emulator-5554 for the first emulator started
178      * with numbers increasing. avdName is as defined in the SDK tool. The
179      * manufacturer is typically "unknown" and the model is typically "sdk".
180      * The deviceid for an actual devices is
181      * deviceSerialNumber_manufacturer_model.
182      * <br><br>
183      * The file contains system properties from the system running
184      * the Android Maven Plugin (JVM) and device properties from the
185      * device/emulator the tests are running on.
186      * <br><br>
187      * The file contains a single TestSuite for all tests and a
188      * TestCase for each test method. Errors and failures are logged
189      * in the file and the system log with full stack traces and other
190      * details available.
191      */
192     @Parameter( property = "android.test.createreport", defaultValue = "true" )
193     private Boolean testCreateReport;
194 
195     /**
196      * <p>Whether to execute tests only in given packages as part of the instrumentation tests.</p>
197      * <pre>
198      * &lt;packages&gt;
199      *     &lt;package&gt;your.package.name&lt;/package&gt;
200      * &lt;/packages&gt;
201      * </pre>
202      * or as e.g. -Dandroid.test.packages=package1,package2
203      */
204     @Parameter( property = "android.test.packages" )
205     protected List<String> testPackages;
206 
207     /**
208      * <p>Whether to execute test classes which are specified as part of the instrumentation tests.</p>
209      * <pre>
210      * &lt;classes&gt;
211      *     &lt;class&gt;your.package.name.YourTestClass&lt;/class&gt;
212      * &lt;/classes&gt;
213      * </pre>
214      * or as e.g. -Dandroid.test.classes=class1,class2
215      */
216     @Parameter( property = "android.test.classes" )
217     protected List<String> testClasses;
218 
219 
220     /**
221      * <p>Whether to execute tests which are annotated with the given annotations.</p>
222      * <pre>
223      * &lt;annotations&gt;
224      *     &lt;annotation&gt;your.package.name.YourAnnotation&lt;/annotation&gt;
225      * &lt;/annotations&gt;
226      * </pre>
227      * or as e.g. -Dandroid.test.annotations=annotation1,annotation2
228      */
229     @Parameter( property = "android.test.annotations" )
230     protected List<String> testAnnotations;
231 
232     /**
233      * <p>Whether to execute tests which are <strong>not</strong> annotated with the given annotations.</p>
234      * <pre>
235      * &lt;excludeAnnotations&gt;
236      *     &lt;excludeAnnotation&gt;your.package.name.YourAnnotation&lt;/excludeAnnotation&gt;
237      * &lt;/excludeAnnotations&gt;
238      * </pre>
239      * or as e.g. -Dandroid.test.excludeAnnotations=annotation1,annotation2
240      */
241     @Parameter( property = "android.test.excludeAnnotations" )
242     protected List<String> testExcludeAnnotations;
243 
244     /**
245      * <p>Extra instrumentation arguments.</p>
246      * <pre>
247      * &lt;instrumentationArgs&gt;
248      *     &lt;instrumentationArg&gt;key value&lt;/instrumentationArg&gt;
249      *     &lt;instrumentationArg&gt;key 'value with spaces'&lt;/instrumentationArg&gt;
250      * &lt;/instrumentationArgs&gt;
251      * </pre>
252      * or as e.g. -Dandroid.test.instrumentationArgs="key1 value1","key2 'value with spaces'"
253      */
254     @Parameter( property = "android.test.instrumentationArgs" )
255     protected List<String> testInstrumentationArgs;
256 
257     private boolean classesExists;
258     private boolean packagesExists;
259 
260     // the parsed parameters from the plugin config or properties from command line or pom or settings
261     private String parsedSkip;
262     private String parsedInstrumentationPackage;
263     private String parsedInstrumentationRunner;
264     private List<String> parsedClasses;
265     private List<String> parsedPackages;
266     private List<String> parsedAnnotations;
267     private List<String> parsedExcludeAnnotations;
268     private Map<String, String> parsedInstrumentationArgs;
269     private String parsedTestSize;
270     private Boolean parsedCoverage;
271     private String parsedCoverageFile;
272     private Boolean parsedDebug;
273     private Boolean parsedLogOnly;
274     private Boolean parsedCreateReport;
275 
276     private String packagesList;
277 
278     protected void instrument() throws MojoExecutionException, MojoFailureException
279     {
280         parseConfiguration();
281 
282         if ( parsedInstrumentationPackage == null )
283         {
284             parsedInstrumentationPackage = extractPackageNameFromAndroidManifest( destinationManifestFile );
285         }
286 
287         if ( parsedInstrumentationRunner == null )
288         {
289             parsedInstrumentationRunner = extractInstrumentationRunnerFromAndroidManifest( destinationManifestFile );
290         }
291 
292         // only run Tests in specific package
293         packagesList = buildCommaSeparatedString( parsedPackages );
294         packagesExists = StringUtils.isNotBlank( packagesList );
295 
296         if ( parsedClasses != null )
297         {
298             classesExists = parsedClasses.size() > 0;
299         }
300         else
301         {
302             classesExists = false;
303         }
304 
305         if ( classesExists && packagesExists )
306         {
307             // if both packages and classes are specified --> ERROR
308             throw new  MojoFailureException( "packages and classes are mutually exclusive. They cannot be specified at"
309                     + " the same time. Please specify either packages or classes. For details, see "
310                     + "http://developer.android.com/guide/developing/testing/testing_otheride.html" );
311         }
312 
313         DeviceCallback instrumentationTestExecutor = new DeviceCallback()
314         {
315             public void doWithDevice( final IDevice device ) throws MojoExecutionException, MojoFailureException
316             {
317                 String deviceLogLinePrefix = DeviceHelper.getDeviceLogLinePrefix( device );
318 
319                 RemoteAndroidTestRunner remoteAndroidTestRunner = new RemoteAndroidTestRunner(
320                         parsedInstrumentationPackage, parsedInstrumentationRunner, device );
321 
322                 if ( packagesExists )
323                 {
324                     for ( String str : packagesList.split( "," ) )
325                     {
326                         remoteAndroidTestRunner.setTestPackageName( str );
327                         getLog().info( deviceLogLinePrefix + "Running tests for specified test package: " + str );
328                     }
329                 }
330 
331                 if ( classesExists )
332                 {
333                     remoteAndroidTestRunner
334                             .setClassNames( parsedClasses.toArray( new String[ parsedClasses.size() ] ) );
335                     getLog().info( deviceLogLinePrefix + "Running tests for specified test classes/methods: " 
336                             + parsedClasses );
337                 }
338 
339                 if ( parsedAnnotations != null )
340                 {
341                     for ( String annotation : parsedAnnotations )
342                     {
343                         remoteAndroidTestRunner.addInstrumentationArg( "annotation", annotation );
344                     }
345                 }
346 
347                 if ( parsedExcludeAnnotations != null )
348                 {
349                     for ( String annotation : parsedExcludeAnnotations )
350                     {
351                         remoteAndroidTestRunner.addInstrumentationArg( "notAnnotation", annotation );
352                     }
353 
354                 }
355 
356                 remoteAndroidTestRunner.setDebug( parsedDebug );
357                 remoteAndroidTestRunner.setCoverage( parsedCoverage );
358                 if ( StringUtils.isNotBlank( parsedCoverageFile ) )
359                 {
360                     remoteAndroidTestRunner.addInstrumentationArg( "coverageFile", parsedCoverageFile );
361                 }
362                 remoteAndroidTestRunner.setLogOnly( parsedLogOnly );
363 
364                 if ( StringUtils.isNotBlank( parsedTestSize ) )
365                 {
366                     IRemoteAndroidTestRunner.TestSize validSize = IRemoteAndroidTestRunner.TestSize
367                             .getTestSize( parsedTestSize );
368                     remoteAndroidTestRunner.setTestSize( validSize );
369                 }
370 
371                 addAllInstrumentationArgs( remoteAndroidTestRunner, parsedInstrumentationArgs );
372 
373                 getLog().info( deviceLogLinePrefix +  "Running instrumentation tests in " 
374                         + parsedInstrumentationPackage );
375                 try
376                 {
377                     AndroidTestRunListener testRunListener = new AndroidTestRunListener( device, getLog(),
378                             parsedCreateReport, false, "", "", targetDirectory );
379                     remoteAndroidTestRunner.run( testRunListener );
380                     if ( testRunListener.hasFailuresOrErrors() && !testFailSafe )
381                     {
382                         throw new MojoFailureException( deviceLogLinePrefix + "Tests failed on device." );
383                     }
384                     if ( testRunListener.testRunFailed() && !testFailSafe  )
385                     {
386                         throw new MojoFailureException( deviceLogLinePrefix + "Test run failed to complete: " 
387                                 + testRunListener.getTestRunFailureCause() );
388                     }
389                     if ( testRunListener.threwException() && !testFailSafe  )
390                     {
391                         throw new MojoFailureException( deviceLogLinePrefix +  testRunListener.getExceptionMessages() );
392                     }
393                 }
394                 catch ( TimeoutException e )
395                 {
396                     throw new MojoExecutionException( deviceLogLinePrefix + "timeout", e );
397                 }
398                 catch ( AdbCommandRejectedException e )
399                 {
400                     throw new MojoExecutionException( deviceLogLinePrefix + "adb command rejected", e );
401                 }
402                 catch ( ShellCommandUnresponsiveException e )
403                 {
404                     throw new MojoExecutionException( deviceLogLinePrefix + "shell command " + "unresponsive", e );
405                 }
406                 catch ( IOException e )
407                 {
408                     throw new MojoExecutionException( deviceLogLinePrefix + "IO problem", e );
409                 }
410             }
411         };
412 
413         instrumentationTestExecutor = new ScreenshotServiceWrapper( instrumentationTestExecutor, project, getLog() );
414 
415         doWithDevices( instrumentationTestExecutor );
416     }
417 
418     private void addAllInstrumentationArgs(
419             final RemoteAndroidTestRunner remoteAndroidTestRunner,
420             final Map<String, String> parsedInstrumentationArgs )
421     {
422         for ( final Map.Entry<String, String> entry : parsedInstrumentationArgs.entrySet() )
423         {
424             remoteAndroidTestRunner.addInstrumentationArg( entry.getKey(), entry.getValue() );
425         }
426     }
427 
428     private void parseConfiguration()
429     {
430         // we got config in pom ... lets use it,
431         if ( test != null )
432         {
433             if ( StringUtils.isNotEmpty( test.getSkip() ) )
434             {
435                 parsedSkip = test.getSkip();
436             }
437             else
438             {
439                 parsedSkip = testSkip;
440             }
441             if ( StringUtils.isNotEmpty( test.getInstrumentationPackage() ) )
442             {
443                 parsedInstrumentationPackage = test.getInstrumentationPackage();
444             }
445             else
446             {
447                 parsedInstrumentationPackage = testInstrumentationPackage;
448             }
449             if ( StringUtils.isNotEmpty( test.getInstrumentationRunner() ) )
450             {
451                 parsedInstrumentationRunner = test.getInstrumentationRunner();
452             }
453             else
454             {
455                 parsedInstrumentationRunner = testInstrumentationRunner;
456             }
457             if ( test.getClasses() != null && ! test.getClasses().isEmpty() )
458             {
459                 parsedClasses = test.getClasses();
460             }
461             else
462             {
463                 parsedClasses = testClasses;
464             }
465             if ( test.getAnnotations() != null && ! test.getAnnotations().isEmpty() )
466             {
467                 parsedAnnotations = test.getAnnotations();
468             }
469             else
470             {
471                 parsedAnnotations =  testAnnotations;
472             }
473             if ( test.getExcludeAnnotations() != null && ! test.getExcludeAnnotations().isEmpty() )
474             {
475                 parsedExcludeAnnotations = test.getExcludeAnnotations();
476             }
477             else
478             {
479                 parsedExcludeAnnotations = testExcludeAnnotations;
480             }
481             if ( test.getPackages() != null && ! test.getPackages().isEmpty() )
482             {
483                 parsedPackages = test.getPackages();
484             }
485             else
486             {
487                 parsedPackages = testPackages;
488             }
489             if ( StringUtils.isNotEmpty( test.getTestSize() ) )
490             {
491                 parsedTestSize = test.getTestSize();
492             }
493             else
494             {
495                 parsedTestSize = testTestSize;
496             }
497             if ( test.isCoverage() != null )
498             {
499                 parsedCoverage = test.isCoverage();
500             }
501             else
502             {
503                 parsedCoverage = testCoverage;
504             }
505             if ( test.getCoverageFile() != null )
506             {
507                 parsedCoverageFile = test.getCoverageFile();
508             }
509             else
510             {
511                 parsedCoverageFile = "";
512             }
513             if ( test.isDebug() != null )
514             {
515                 parsedDebug = test.isDebug();
516             }
517             else
518             {
519                 parsedDebug = testDebug;
520             }
521             if ( test.isLogOnly() != null )
522             {
523                 parsedLogOnly = test.isLogOnly();
524             }
525             else
526             {
527                 parsedLogOnly = testLogOnly;
528             }
529             if ( test.isCreateReport() != null )
530             {
531                 parsedCreateReport = test.isCreateReport();
532             }
533             else
534             {
535                 parsedCreateReport = testCreateReport;
536             }
537 
538             parsedInstrumentationArgs = InstrumentationArgumentParser.parse( test.getInstrumentationArgs() );
539         }
540         // no pom, we take properties
541         else
542         {
543             parsedSkip = testSkip;
544             parsedInstrumentationPackage = testInstrumentationPackage;
545             parsedInstrumentationRunner = testInstrumentationRunner;
546             parsedClasses = testClasses;
547             parsedAnnotations = testAnnotations;
548             parsedExcludeAnnotations = testExcludeAnnotations;
549             parsedPackages = testPackages;
550             parsedTestSize = testTestSize;
551             parsedCoverage = testCoverage;
552             parsedCoverageFile = testCoverageFile;
553             parsedDebug = testDebug;
554             parsedLogOnly = testLogOnly;
555             parsedCreateReport = testCreateReport;
556             parsedInstrumentationArgs = InstrumentationArgumentParser.parse( testInstrumentationArgs );
557         }
558     }
559 
560     /**
561      * Whether or not to execute integration test related goals. Reads from configuration parameter
562      * <code>enableIntegrationTest</code>, but can be overridden with <code>-Dmaven.test.skip</code>.
563      *
564      * @return <code>true</code> if integration test goals should be executed, <code>false</code> otherwise.
565      */
566     protected boolean isEnableIntegrationTest() throws MojoFailureException, MojoExecutionException
567     {
568         parseConfiguration();
569         if ( mavenTestSkip )
570         {
571             getLog().info( "maven.test.skip set - skipping tests" );
572             return false;
573         }
574 
575         if ( mavenSkipTests )
576         {
577             getLog().info( "maven.skip.tests set - skipping tests" );
578             return false;
579         }
580 
581         if ( "true".equalsIgnoreCase( parsedSkip ) )
582         {
583             getLog().info( "android.test.skip set - skipping tests" );
584             return false;
585         }
586 
587         if ( "false".equalsIgnoreCase( parsedSkip ) )
588         {
589             return true;
590         }
591 
592         if ( parsedSkip == null || "auto".equalsIgnoreCase( parsedSkip ) )
593         {
594             if ( extractInstrumentationRunnerFromAndroidManifest( destinationManifestFile ) == null )
595             {
596                 getLog().info( "No InstrumentationRunner found - skipping tests" );
597                 return false;
598             }
599             return AndroidTestFinder.containsAndroidTests( projectOutputDirectory );
600         }
601 
602         throw new MojoFailureException( "android.test.skip must be configured as 'true', 'false' or 'auto'." );
603 
604     }
605 
606     /**
607      * Helper method to build a comma separated string from a list.
608      * Blank strings are filtered out
609      *
610      * @param lines A list of strings
611      * @return Comma separated String from given list
612      */
613     protected static String buildCommaSeparatedString( List<String> lines )
614     {
615         if ( lines == null || lines.size() == 0 )
616         {
617             return null;
618         }
619 
620         List<String> strings = new ArrayList<String>( lines.size() );
621         for ( String str : lines )
622         { // filter out blank strings
623             if ( StringUtils.isNotBlank( str ) )
624             {
625                 strings.add( StringUtils.trimToEmpty( str ) );
626             }
627         }
628 
629         return StringUtils.join( strings, "," );
630     }
631 
632 }