View Javadoc
1   package com.simpligility.maven.plugins.android;
2   
3   import com.android.ddmlib.AdbCommandRejectedException;
4   import com.android.ddmlib.IDevice;
5   import com.android.ddmlib.IShellOutputReceiver;
6   import com.android.ddmlib.ShellCommandUnresponsiveException;
7   import com.android.ddmlib.TimeoutException;
8   import com.android.ddmlib.testrunner.ITestRunListener;
9   import com.android.ddmlib.testrunner.TestIdentifier;
10  import com.simpligility.maven.plugins.android.common.DeviceHelper;
11  
12  import org.apache.commons.io.FileUtils;
13  import org.apache.commons.lang3.StringUtils;
14  import org.apache.maven.plugin.logging.Log;
15  import org.apache.maven.surefire.ObjectFactory;
16  import org.apache.maven.surefire.Testsuite;
17  
18  import javax.xml.bind.JAXBContext;
19  import javax.xml.bind.JAXBException;
20  import javax.xml.bind.Marshaller;
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.text.DecimalFormat;
25  import java.text.NumberFormat;
26  import java.util.Map;
27  
28  /**
29   * AndroidTestRunListener produces a nice output for the log for the test run as well as an xml file compatible with
30   * the junit xml report file format understood by many tools.
31   * 
32   * It will do so for each device/emulator the tests run on.
33   */
34  public class AndroidTestRunListener implements ITestRunListener
35  {
36      private static final String SCREENSHOT_SUFFIX = "_screenshot.png";
37  
38      /**
39       * the indent used in the log to group items that belong together visually *
40       */
41      private static final String INDENT = "  ";
42  
43      /**
44       * time format for the output of milliseconds in seconds in the xml file *
45       */
46      private final NumberFormat timeFormatter = new DecimalFormat( "#0.000" );
47  
48      private int testCount = 0;
49      private int testRunCount = 0;
50      private int testIgnoredCount = 0;
51      private int testFailureCount = 0;
52      private int testErrorCount = 0;
53      private String testRunFailureCause = null;
54  
55      /**
56       * the emulator or device we are running the tests on *
57       */
58      private final IDevice device;
59      private final Log log;
60      private final Boolean createReport;
61      private final Boolean takeScreenshotOnFailure;
62      private final String screenshotsPathOnDevice;
63      private final String reportSuffix;
64      private final File targetDirectory;
65  
66      private final String deviceLogLinePrefix;
67  
68      private final ObjectFactory objectFactory = new ObjectFactory();
69      private Testsuite report;
70      private Testsuite.Testcase currentTestCase;
71  
72      /**
73       * start time of current test case in millis, reset with each test start
74       */
75      private long currentTestCaseStartTime;
76  
77      // we track if we have problems and then report upstream
78      private boolean threwException = false;
79      private final StringBuilder exceptionMessages = new StringBuilder();
80  
81      /**
82       * Create a new test run listener.
83       *
84       * @param device
85       *            the device on which test is executed.
86       */
87      public AndroidTestRunListener( IDevice device, Log log, Boolean createReport,
88                                     Boolean takeScreenshotOnFailure, String screenshotsPathOnDevice,
89                                     String reportSuffix, File targetDirectory )
90      {
91          this.device = device;
92          this.deviceLogLinePrefix = DeviceHelper.getDeviceLogLinePrefix( device );
93          this.log = log;
94          this.createReport = createReport;
95          this.takeScreenshotOnFailure = takeScreenshotOnFailure;
96          this.screenshotsPathOnDevice = screenshotsPathOnDevice;
97          this.reportSuffix = reportSuffix;
98          this.targetDirectory = targetDirectory;
99      }
100 
101     public Log getLog()
102     {
103         return this.log;
104     }
105 
106     @Override
107     public void testRunStarted( String runName, int tCount )
108     {
109         if ( takeScreenshotOnFailure )
110         {
111             executeOnAdbShell( "rm -f " + screenshotsPathOnDevice + "/*screenshot.png" );
112             executeOnAdbShell( "mkdir " + screenshotsPathOnDevice );
113         }
114 
115         this.testCount = tCount;
116         getLog().info( deviceLogLinePrefix + INDENT + "Run started: " + runName + ", " + testCount + " tests:" );
117 
118         if ( createReport )
119         {
120             report = new Testsuite();
121             report.setName( runName );
122             final Testsuite.Properties props = new Testsuite.Properties();
123             report.getProperties().add( props );
124             for ( Map.Entry< Object, Object > systemProperty : System.getProperties().entrySet() )
125             {
126                 final Testsuite.Properties.Property property = new Testsuite.Properties.Property();
127                 property.setName( systemProperty.getKey().toString() );
128                 property.setValue( systemProperty.getValue().toString() );
129                 props.getProperty().add( property );
130             }
131             Map< String, String > deviceProperties = device.getProperties();
132             for ( Map.Entry< String, String > deviceProperty : deviceProperties.entrySet() )
133             {
134                 final Testsuite.Properties.Property property = new Testsuite.Properties.Property();
135                 property.setName( deviceProperty.getKey() );
136                 property.setValue( deviceProperty.getValue() );
137                 props.getProperty().add( property );
138             }
139         }
140     }
141 
142     @Override
143     public void testIgnored( TestIdentifier testIdentifier )
144     {
145         ++testIgnoredCount;
146 
147         getLog().info( deviceLogLinePrefix + INDENT + INDENT + testIdentifier.toString() );
148 
149     }
150 
151     @Override
152     public void testStarted( TestIdentifier testIdentifier )
153     {
154         testRunCount++;
155         getLog().info(
156                 deviceLogLinePrefix
157                         + String.format( "%1$s%1$sStart [%2$d/%3$d]: %4$s", INDENT, testRunCount, testCount,
158                         testIdentifier.toString() ) );
159 
160         if ( createReport )
161         { // reset start time for each test run
162             currentTestCaseStartTime = System.currentTimeMillis();
163             currentTestCase = new Testsuite.Testcase();
164             currentTestCase.setClassname( testIdentifier.getClassName() );
165             currentTestCase.setName( testIdentifier.getTestName() );
166         }
167     }
168 
169     @Override
170     public void testFailed( TestIdentifier testIdentifier, String trace )
171     {
172         if ( takeScreenshotOnFailure )
173         {
174             String suffix = "_error";
175             String filepath = testIdentifier.getTestName() + suffix + SCREENSHOT_SUFFIX;
176 
177             executeOnAdbShell( "screencap -p " + screenshotsPathOnDevice + "/" + filepath );
178             getLog().info( deviceLogLinePrefix + INDENT + INDENT + filepath + " saved." );
179         }
180 
181         ++testErrorCount;
182 
183         getLog().info( deviceLogLinePrefix + INDENT + INDENT + testIdentifier.toString() );
184         getLog().info( deviceLogLinePrefix + INDENT + INDENT + trace );
185 
186         if ( createReport )
187         {
188             final Testsuite.Testcase.Error error = new Testsuite.Testcase.Error();
189             error.setValue( trace );
190             error.setMessage( parseForMessage( trace ) );
191             error.setType( parseForException( trace ) );
192             currentTestCase.setError( objectFactory.createTestsuiteTestcaseError( error ) );
193         }
194     }
195 
196     @Override
197     public void testAssumptionFailure( TestIdentifier testIdentifier, String trace )
198     {
199         if ( takeScreenshotOnFailure )
200         {
201             String suffix = "_failure";
202             String filepath = testIdentifier.getTestName() + suffix + SCREENSHOT_SUFFIX;
203 
204             executeOnAdbShell( "screencap -p " + screenshotsPathOnDevice + "/" + filepath );
205             getLog().info( deviceLogLinePrefix + INDENT + INDENT + filepath + " saved." );
206         }
207 
208         ++testFailureCount;
209 
210         getLog().info( deviceLogLinePrefix + INDENT + INDENT + testIdentifier.toString() );
211         getLog().info( deviceLogLinePrefix + INDENT + INDENT + trace );
212 
213         if ( createReport )
214         {
215             final Testsuite.Testcase.Failure failure = new Testsuite.Testcase.Failure();
216             failure.setValue( trace );
217             failure.setMessage( parseForMessage( trace ) );
218             failure.setType( parseForException( trace ) );
219             currentTestCase.getFailure().add( failure );
220         }
221     }
222 
223     private void executeOnAdbShell( String command )
224     {
225         try
226         {
227             device.executeShellCommand( command, new IShellOutputReceiver()
228             {
229                 @Override
230                 public boolean isCancelled()
231                 {
232                     return false;
233                 }
234 
235                 @Override
236                 public void flush()
237                 {
238                 }
239 
240                 @Override
241                 public void addOutput( byte[] data, int offset, int length )
242                 {
243                 }
244             } );
245         }
246         catch ( TimeoutException | AdbCommandRejectedException | IOException | ShellCommandUnresponsiveException e )
247         {
248             getLog().error( e );
249         }
250     }
251 
252     @Override
253     public void testEnded( TestIdentifier testIdentifier, Map< String, String > testMetrics )
254     {
255         getLog().info(
256                 deviceLogLinePrefix
257                         + String.format( "%1$s%1$sEnd [%2$d/%3$d]: %4$s", INDENT, testRunCount, testCount,
258                         testIdentifier.toString() ) );
259         logMetrics( testMetrics );
260 
261         if ( createReport )
262         {
263             double seconds = ( System.currentTimeMillis() - currentTestCaseStartTime ) / 1000.0;
264             currentTestCase.setTime( timeFormatter.format( seconds ) );
265             report.getTestcase().add( currentTestCase );
266         }
267     }
268 
269     @Override
270     public void testRunEnded( long elapsedTime, Map< String, String > runMetrics )
271     {
272         getLog().info( deviceLogLinePrefix + INDENT + "Run ended: " + elapsedTime + " ms" );
273         if ( hasFailuresOrErrors() )
274         {
275             getLog().error( deviceLogLinePrefix + INDENT + "FAILURES!!!" );
276         }
277         getLog().info(
278                 INDENT + "Tests run: " + testRunCount
279                         + ( testRunCount < testCount ? " (of " + testCount + ")" : "" ) + ",  Failures: "
280                         + testFailureCount + ",  Errors: " + testErrorCount
281                         + ",  Ignored: " + testIgnoredCount );
282 
283         if ( createReport )
284         {
285             report.setTests( Integer.toString( testCount ) );
286             report.setFailures( Integer.toString( testFailureCount ) );
287             report.setErrors( Integer.toString( testErrorCount ) );
288             report.setSkipped( Integer.toString( testIgnoredCount ) );
289             report.setTime( timeFormatter.format( elapsedTime / 1000.0 ) );
290         }
291 
292         logMetrics( runMetrics );
293 
294         if ( createReport )
295         {
296             writeJunitReportToFile();
297         }
298     }
299 
300     @Override
301     public void testRunFailed( String errorMessage )
302     {
303         testRunFailureCause = errorMessage;
304         getLog().info( deviceLogLinePrefix + INDENT + "Run failed: " + errorMessage );
305     }
306 
307     @Override
308     public void testRunStopped( long elapsedTime )
309     {
310         getLog().info( deviceLogLinePrefix + INDENT + "Run stopped:" + elapsedTime );
311     }
312 
313     /**
314      * Parse a trace string for the message in it. Assumes that the message is located after ":" and before "\r\n".
315      *
316      * @param trace stack trace from android tests
317      * @return message or empty string
318      */
319     private String parseForMessage( String trace )
320     {
321         if ( StringUtils.isNotBlank( trace ) )
322         {
323             String newline = "\r\n";
324             // if there is message like
325             // junit.junit.framework.AssertionFailedError ... there is no
326             // message
327             int messageEnd = trace.indexOf( newline );
328             boolean hasMessage = !trace.startsWith( "junit." ) && messageEnd > 0;
329             if ( hasMessage )
330             {
331                 int messageStart = trace.indexOf( ":" ) + 2;
332                 if ( messageStart > messageEnd )
333                 {
334                     messageEnd = trace.indexOf( newline + "at" );
335                     // match start of stack trace "\r\nat org.junit....."
336                     if ( messageStart > messageEnd )
337                     {
338                         // ':' wasn't found in message but in stack trace
339                         messageStart = 0;
340                     }
341                 }
342                 return trace.substring( messageStart, messageEnd );
343             }
344             else
345             {
346                 return StringUtils.EMPTY;
347             }
348         }
349         else
350         {
351             return StringUtils.EMPTY;
352         }
353     }
354 
355     /**
356      * Parse a trace string for the exception class. Assumes that it is the start of the trace and ends at the first
357      * ":".
358      *
359      * @param trace stack trace from android tests
360      * @return Exception class as string or empty string
361      */
362     private String parseForException( String trace )
363     {
364         if ( StringUtils.isNotBlank( trace ) )
365         {
366             return trace.substring( 0, trace.indexOf( ":" ) );
367         }
368         else
369         {
370             return StringUtils.EMPTY;
371         }
372     }
373 
374     /**
375      * Write the junit report xml file.
376      */
377     private void writeJunitReportToFile()
378     {
379         try
380         {
381             final String directory = String.valueOf( targetDirectory ) + "/surefire-reports";
382             FileUtils.forceMkdir( new File ( directory ) );
383             final StringBuilder b = new StringBuilder( directory ).append( "/TEST-" )
384                     .append( DeviceHelper.getDescriptiveName( device ) );
385 
386             if ( StringUtils.isNotBlank( reportSuffix ) )
387             {
388                 //Safety first
389                 b.append( reportSuffix.replace( "/", "" ).replace( "\\", "" ) );
390             }
391 
392             final File reportFile = new File( b.append( ".xml" ).toString() );
393             final JAXBContext jaxbContext = JAXBContext.newInstance( ObjectFactory.class );
394             final Marshaller marshaller = jaxbContext.createMarshaller();
395             marshaller.marshal( report, reportFile );
396 
397             getLog().info( deviceLogLinePrefix + "Report file written to " + reportFile.getAbsolutePath() );
398         }
399         catch ( IOException e )
400         {
401             threwException = true;
402             exceptionMessages.append( "Failed to write test report file" );
403             exceptionMessages.append( e.getMessage() );
404         }
405         catch ( JAXBException e )
406         {
407             threwException = true;
408             exceptionMessages.append( "Failed to create jaxb context" );
409             exceptionMessages.append( e.getMessage() );
410         }
411     }
412 
413     /**
414      * Log all the metrics out in to key: value lines.
415      *
416      * @param metrics key-value pairs reported at the end of a test run
417      */
418     private void logMetrics( Map< String, String > metrics )
419     {
420         for ( Map.Entry< String, String > entry : metrics.entrySet() )
421         {
422             getLog().info( deviceLogLinePrefix + INDENT + INDENT + entry.getKey() + ": " + entry.getValue() );
423         }
424     }
425 
426     /**
427      * @return if any failures or errors occurred in the test run.
428      */
429     public boolean hasFailuresOrErrors()
430     {
431         return testErrorCount > 0 || testFailureCount > 0;
432     }
433 
434     /**
435      * @return if the test run itself failed - a failure in the test infrastructure, not a test failure.
436      */
437     public boolean testRunFailed()
438     {
439         return testRunFailureCause != null;
440     }
441 
442     /**
443      * @return the cause of test failure if any.
444      */
445     public String getTestRunFailureCause()
446     {
447         return testRunFailureCause;
448     }
449 
450     /**
451      * @return if any exception was thrown during the test run on the build system (not the Android device or
452      *         emulator)
453      */
454     public boolean threwException()
455     {
456         return threwException;
457     }
458 
459     /**
460      * @return all exception messages thrown during test execution on the test run time (not the Android device or
461      *         emulator)
462      */
463     public String getExceptionMessages()
464     {
465         return exceptionMessages.toString();
466     }
467 }