View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  
27  import com.eviware.soapui.SoapUI;
28  import com.eviware.soapui.impl.wsdl.WsdlProject;
29  import com.eviware.soapui.impl.wsdl.support.assertions.Assertable.AssertionStatus;
30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
31  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
32  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
33  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
35  import com.eviware.soapui.model.iface.Attachment;
36  import com.eviware.soapui.model.support.PropertiesMap;
37  import com.eviware.soapui.model.testsuite.TestCase;
38  import com.eviware.soapui.model.testsuite.TestRunContext;
39  import com.eviware.soapui.model.testsuite.TestRunListener;
40  import com.eviware.soapui.model.testsuite.TestRunner;
41  import com.eviware.soapui.model.testsuite.TestStep;
42  import com.eviware.soapui.model.testsuite.TestStepResult;
43  import com.eviware.soapui.model.testsuite.TestSuite;
44  import com.eviware.soapui.model.testsuite.TestRunner.Status;
45  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
46  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
47  import com.eviware.soapui.report.JUnitReportCollector;
48  import com.eviware.soapui.support.Tools;
49  
50  /***
51   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
52   * directly from other classes.
53   * <p>
54   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
55   * calling run</p> 
56   * 
57   * @author Ole.Matzura
58   */
59  
60  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
61  {
62  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
63  	
64  	private String testSuite;
65  	private String testCase;
66  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
67  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
68  	private List<TestCase> runningTests = new ArrayList<TestCase>();
69  	private List<TestCase> failedTests = new ArrayList<TestCase>();
70  	private String endpoint;
71  	private String domain;
72  	private String password;
73  	private String username;
74  	private String host;
75  	
76  	private int testSuiteCount;
77  	private int testCaseCount;
78  	private int testStepCount;
79  	private int testAssertionCount;
80  
81  	private boolean printReport;
82  	private String outputFolder;
83  	private boolean exportAll;
84  	private boolean junitReport;
85  	private int exportCount;
86  	private JUnitReportCollector reportCollector;
87  	private String wssPasswordType;
88  	private WsdlProject project;
89  	
90  	/***
91  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
92  	 * 
93  	 * @param args
94  	 * @throws Exception
95  	 */
96  
97  	@SuppressWarnings("static-access")
98  	public static void main( String [] args) throws Exception
99  	{
100 		new SoapUITestCaseRunner().runFromCommandLine( args );
101 	}
102 	
103 	protected boolean processCommandLine( CommandLine cmd )
104 	{
105 		if( cmd.hasOption( "e"))
106 			setEndpoint( cmd.getOptionValue( "e" ) );
107 		
108 		if( cmd.hasOption( "s"))
109 			setTestSuite( cmd.getOptionValue( "s") );
110 
111 		if( cmd.hasOption( "c"))
112 			setTestCase( cmd.getOptionValue( "c") );
113 
114 		if( cmd.hasOption( "u"))
115 			setUsername( cmd.getOptionValue( "u") );
116 
117 		if( cmd.hasOption( "p"))
118 			setPassword( cmd.getOptionValue( "p") );
119 		
120 		if( cmd.hasOption( "w"))
121 			setWssPasswordType( cmd.getOptionValue( "w") );
122 
123 		if( cmd.hasOption( "d"))
124 			setDomain( cmd.getOptionValue( "d") );
125 
126 		if( cmd.hasOption( "h"))
127 			setHost( cmd.getOptionValue( "h") );
128 		
129 		if( cmd.hasOption( "f"))
130 			setOutputFolder( cmd.getOptionValue( "f") );
131 		
132 		if( cmd.hasOption( "t"))
133 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
134 		
135 		if( cmd.hasOption( "i"))
136 		{
137 			enableSwingUI();
138 		}
139 		
140 		setPrintReport( cmd.hasOption( "r" ) );
141 		setExportAll( cmd.hasOption( "a" ) );
142 		setJUnitReport( cmd.hasOption( "j" ) );
143 		
144 		return true;
145 	}
146 
147 	protected SoapUIOptions initCommandLineOptions()
148 	{
149 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
150 		options.addOption( "e", true, "Sets the endpoint" );
151 		options.addOption( "s", true, "Sets the testsuite" );
152 		options.addOption( "c", true, "Sets the testcase" );
153 		options.addOption( "u", true, "Sets the username" );
154 		options.addOption( "p", true, "Sets the password" );
155 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
156 		options.addOption( "i", false, "Enables Swing UI for scripts" );
157 		options.addOption( "d", true, "Sets the domain" );
158 		options.addOption( "h", true, "Sets the host" );
159 		options.addOption( "r", false, "Prints a small summary report" );
160 		options.addOption( "f", true, "Sets the output folder to export results to" );
161 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
162 		options.addOption( "a", false, "Turns on exporting of all results" );
163 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
164 		return options;
165 	}
166 
167 	/***
168 	 * Add console appender to groovy log
169 	 */
170 	
171 	
172 	public void setExportAll(boolean exportAll)
173 	{
174 		this.exportAll = exportAll;
175 	}
176 
177 	public void setJUnitReport(boolean junitReport)
178 	{
179 		this.junitReport = junitReport;
180 		if (junitReport)
181 			reportCollector = new JUnitReportCollector();
182 	}
183 	
184 	public void setOutputFolder(String outputFolder)
185 	{
186 		this.outputFolder = outputFolder;
187 	}
188 	
189 	public SoapUITestCaseRunner()
190 	{
191 		super( SoapUITestCaseRunner.TITLE  );
192 	}
193 	
194 	public SoapUITestCaseRunner( String title )
195 	{
196 		super( title  );
197 	}
198 	
199 	/***
200 	 * Controls if a short test summary should be printed after the test runs
201 	 * 
202 	 * @param printReport a flag controlling if a summary should be printed
203 	 */
204 	
205 	public void setPrintReport(boolean printReport)
206 	{
207 		this.printReport = printReport;
208 	}
209 
210 	/***
211 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
212 	 * 
213 	 * @param host the host to use by all requests
214 	 */
215 
216 	public void setHost(String host)
217 	{
218 		this.host = host;
219 	}
220 
221 	/***
222 	 * Sets the domain to use for any authentications
223 	 * 
224 	 * @param domain the domain to use for any authentications
225 	 */
226 	
227 	public void setDomain(String domain)
228 	{
229 		log.info( "Setting domain to [" + domain + "]" );
230 		this.domain = domain;
231 	}
232 
233 	/***
234 	 * Sets the password to use for any authentications
235 	 * 
236 	 * @param domain the password to use for any authentications
237 	 */
238 	
239 	public void setPassword(String password)
240 	{
241 		log.info( "Setting password to [" + password + "]" );
242 		this.password = password;
243 	}
244 	
245 	/***
246 	 * Sets the WSS password-type to use for any authentications. Setting this will result
247 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
248 	 * the specified username and password.
249 	 * 
250 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
251 	 */
252 	
253 	public void setWssPasswordType( String wssPasswordType )
254 	{
255 		this.wssPasswordType = wssPasswordType;
256 	}
257 
258 	/***
259 	 * Sets the username to use for any authentications
260 	 * 
261 	 * @param domain the username to use for any authentications
262 	 */
263 	
264 	public void setUsername(String username)
265 	{
266 		log.info( "Setting username to [" + username + "]" );
267 		this.username = username;
268 	}
269 	
270 	/***
271 	 * Runs the testcases as configured with setXXX methods
272 	 * 
273 	 * @throws Exception thrown if any tests fail
274 	 */
275 
276 	public void run() throws Exception
277 	{
278 		initGroovyLog();
279 		
280 		if( outputFolder != null )
281 		{
282 			//	ensure folder exists
283 			File folder = new File( outputFolder );
284 			if( !folder.exists())
285 				folder.mkdirs();
286 		}
287 		
288 		assertions.clear();
289 		
290 		String projectFile = getProjectFile();
291 		
292 		if( !new File( projectFile ).exists() )
293 			throw new Exception( "soapUI project file [" + projectFile + "] not found" );
294 		
295 		project = new WsdlProject( projectFile, null );
296 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
297 		
298 		long startTime = System.nanoTime();
299 		
300 		// start by listening to all testcases.. (since one testcase can call another)
301 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
302 		{
303 			TestSuite suite = project.getTestSuiteAt( c );
304 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
305 			{
306 				suite.getTestCaseAt( i ).addTestRunListener( this );
307 				if (junitReport)
308 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
309 			}
310 		}
311 		
312 		// now run tests..
313 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
314 		{
315 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
316 			{
317 				runSuite( project.getTestSuiteAt( c ));
318 				testSuiteCount++;
319 				
320 				//	wait for tests to finish if running in parallell mode
321 				if( !runningTests.isEmpty() )
322 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
323 				
324 				while( !runningTests.isEmpty() )
325 				{
326 					Thread.sleep( 100 );
327 				}
328 			}
329 		}
330 		
331 		long timeTaken = (System.nanoTime()-startTime)/1000000;
332 		
333 		if( printReport )
334 		{
335 			printReport( timeTaken );
336 		}
337 		
338 		if (junitReport) {
339 			exportJUnitReports( reportCollector, outputFolder );
340 		}
341 		
342 		if( assertions.size() > 0 || failedTests.size() > 0 )
343 		{
344 			throwFailureException();
345 		}
346 	}
347 
348 	protected void throwFailureException() throws Exception
349 	{
350 		StringBuffer buf = new StringBuffer();
351 		
352 		for( int c = 0; c < assertions.size(); c++ )
353 		{
354 			WsdlMessageAssertion assertion = assertions.get( c );
355 			WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
356 			failedTests.remove( testRequest.getTestCase() );
357 			
358 			buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
359 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
360 			
361 			WsdlTestStepResult result = assertionResults.get( assertion  );
362 			StringWriter stringWriter = new StringWriter();
363 			PrintWriter writer = new PrintWriter( stringWriter );
364 			result.writeTo( writer );
365 			buf.append( stringWriter.toString() );
366 		}
367 
368 		while( !failedTests.isEmpty() )
369 		{
370 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
371 		}
372 		
373 		throw new Exception( buf.toString() );
374 	}
375 
376 	public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
377 	{
378 		collector.saveReports(folder == null ? "" : folder);
379 	}
380 
381 	public void printReport( long timeTaken )
382 	{
383 		System.out.println();
384 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
385 		System.out.println( "-----------------------------" );
386 		System.out.println( "Time Taken: " + timeTaken + "ms" );
387 		System.out.println( "Total TestSuites: " + testSuiteCount );
388 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
389 		System.out.println( "Total TestSteps: " + testStepCount );
390 		System.out.println( "Total Request Assertions: " + testAssertionCount );
391 		System.out.println( "Total Failed Assertions: " + assertions.size() );
392 		System.out.println( "Total Exported Results: " + exportCount );
393 	}
394 
395 	/***
396 	 * Run tests in the specified TestSuite
397 	 *
398 	 * @param suite the TestSuite to run
399 	 */
400 	
401 	public void runSuite(TestSuite suite)
402 	{
403 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
404 		long start = System.currentTimeMillis();
405 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
406 		{
407 			String name = suite.getTestCaseAt( c ).getName();
408 			if( testCase == null || name.equalsIgnoreCase( testCase ))
409 			{
410 				runTestCase( suite.getTestCaseAt( c ));
411 			}
412 			else
413 			{
414 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
415 			}
416 		}
417 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
418 	}
419 
420 	/***
421 	 * Runs the specified TestCase
422 	 * 
423 	 * @param testCase the testcase to run
424 	 */
425 	
426 	private void runTestCase(TestCase testCase)
427 	{
428 		runningTests.add( testCase );
429 		testCase.run( PropertiesMap.EMPTY_MAP,	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
430 	}
431 
432 	/***
433 	 * Sets the testcase to run
434 	 * 
435 	 * @param testCase the testcase to run
436 	 */
437 	
438 	public void setTestCase(String testCase)
439 	{
440 		log.info( "setting testCase to [" + testCase + "]" );
441       this.testCase = testCase;
442 	}
443 	
444 	/***
445 	 * Sets the endpoint to use for all test requests
446 	 * 
447 	 * @param endpoint the endpoint to use for all test requests
448 	 */
449 	
450 	public void setEndpoint(String endpoint)
451 	{
452 		log.info( "setting test endpoint to [" + endpoint+ "]" );
453 		this.endpoint = endpoint.trim();
454 	}
455 	
456 	/***
457 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
458 	 * 
459 	 * @param testSuite the testSuite to run.
460 	 */
461 
462 	public void setTestSuite(String testSuite)
463 	{
464 	   log.info( "setting testSuite to [" + testSuite + "]" );
465 		this.testSuite = testSuite;
466 	}
467 	
468 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
469 	{
470 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
471 	}
472 
473 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
474 	{
475 		TestStep currentStep = runContext.getCurrentStep();
476 		log.info( "running step [" + currentStep.getName() + "]" );
477 		
478 		if( currentStep instanceof WsdlTestRequestStep )
479 		{
480 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
481 			if( endpoint != null && endpoint.length() > 0 )
482 			{
483 				requestStep.getTestRequest().setEndpoint( endpoint );
484 			}
485 			
486 			if( host != null && host.length() > 0 )
487 			{
488 				try
489 				{
490 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
491 					requestStep.getTestRequest().setEndpoint( ep );
492 				}
493 				catch (Exception e)
494 				{
495 					log.error( "Failed to set host on endpoint", e );
496 				}				
497 			}
498 
499 			if( username != null && username.length() > 0 )
500 			{
501 				requestStep.getTestRequest().setUsername( username );
502 			}
503 			
504 			if( password != null && password.length() > 0 )
505 			{
506 				requestStep.getTestRequest().setPassword( password );
507 			}
508 			
509 			if( domain != null && domain.length() > 0 )
510 			{
511 				requestStep.getTestRequest().setDomain( domain );
512 			}
513 			
514 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
515 			{
516 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
517 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
518 			}
519 		}
520 	}
521 
522 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
523 	{
524 		TestStep currentStep = runContext.getCurrentStep();
525 		
526 		if( currentStep instanceof WsdlTestRequestStep )
527 		{
528 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
529 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
530 			{
531 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
532 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
533 				if( assertion.getStatus() == AssertionStatus.FAILED )
534 				{
535 					log.info( "ASSERTION FAILED -> " + assertion.getErrors());
536 					assertions.add( assertion );
537 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
538 				}
539 				
540 				testAssertionCount++;
541 			}
542 		}	
543 		
544 		String countPropertyName = currentStep.getName() + " run count";
545 		Long count = (Long) runContext.getProperty( countPropertyName );
546 		if( count == null )
547 		{
548 			count = new Long( 0 );
549 		}
550 
551 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
552 		
553 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
554 		{
555 			try
556 			{
557 				String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
558 														  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +	
559 														  count.longValue() + "-" + result.getStatus();
560 				
561 				String fileName = nameBase + ".txt";
562 				if( outputFolder != null )
563 				{
564 					fileName = outputFolder + File.separator + fileName;
565 				}
566 				
567 				if( result.getStatus() == TestStepStatus.FAILED )
568 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
569 				
570 				PrintWriter writer = new PrintWriter(fileName);
571 				result.writeTo(writer);
572 				writer.close();
573 				
574 				// write attachments
575 				if( result instanceof WsdlTestRequestStepResult )
576 				{
577 					Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
578 					if( attachments != null && attachments.length > 0 )
579 					{
580 						for( int c = 0; c < attachments.length; c++ )
581 						{
582 							fileName = nameBase + "-attachment-" + (c+1) + ".";
583 							
584 							Attachment attachment = attachments[c];
585 							String contentType = attachment.getContentType();
586 							if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
587 							{
588 								fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
589 							}
590 							else
591 							{
592 								fileName += "dat";
593 							}
594 							
595 							if( outputFolder != null )
596 							{
597 								fileName = outputFolder + File.separator + fileName;
598 							}
599 							
600 							FileOutputStream outFile = new FileOutputStream( fileName );
601 							Tools.writeAll( outFile, attachment.getInputStream() );
602 							outFile.close();
603 						}
604 					}
605 				}
606 				
607 				exportCount++;
608 			}
609 			catch (Exception e)
610 			{
611 				log.error( "Error saving failed result: " + e, e );
612 			}			
613 		}
614 		
615 		testStepCount++;
616 	}
617 
618 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
619 	{
620 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " + 
621 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
622 		
623 		if( testRunner.getStatus() == Status.FAILED )
624 		{
625 			failedTests.add( testRunner.getTestCase() );
626 		}
627 		
628 		runningTests.remove( testRunner.getTestCase() );
629 		
630 		testCaseCount++;
631 	}
632 
633 	protected WsdlProject getProject()
634 	{
635 		return project;
636 	}
637 }