Merge pull request #627 from Codeception/multi-reports

Multi reports and Mochawesome helper

Author: APshenkin

CHANGELOG.md

@@ -41,7 +41,7 @@ Next notable feature is **[SmartWait](http://codecept.io/acceptance/#smartwait)*
 
 * Read more about [SmartWait](http://codecept.io/acceptance/#smartwait)
 
-#### Changelog
+**Changelog**
 
 * Minimal NodeJS version is 6.11.1 LTS
 * Use `within` command with generators.
@@ -486,4 +486,4 @@ Whenever you need to create `I` object (in page objects, custom steps, but not i
 * [WebDriverIO] proxy configuration added by @petehouston
 * [WebDriverIO] fixed `waitForText` method by @roadhump. Fixes #11
 * Fixed creating output dir when it already exists on init by @alfirin
-* Fixed loading of custom helpers
+* Fixed loading of custom helpers

docs/helpers/Appium.md

@@ -23,6 +23,8 @@ This helper should be configured in codecept.json or codecept.conf.js
 
 -   `port`: Appium port
 -   `restart`: restart browser or app between tests (default: true), if set to false cookies will be cleaned but browser window will be kept and for apps nothing will be changed.
+-   `disableScreenshots` (optional, default: false)  - don't save screenshot on failure
+-   `uniqueScreenshotNames` (optional, default: false)  - option to prevent screenshot override if you have scenarios with the same name in different suites
 -   `desiredCapabilities`: [], Appium capabilities, see below
     -   `platformName` - Which mobile OS platform to use
     -   `appPackage` - Java package of the Android app you want to run

docs/helpers/Nightmare.md

@@ -13,6 +13,9 @@ This helper should be configured in codecept.json
 
 -   `url` - base url of website to be tested
 -   `restart` (optional, default: true) - restart browser between tests.
+-   `disableScreenshots` (optional, default: false)  - don't save screenshot on failure
+-   `uniqueScreenshotNames` (optional, default: false)  - option to prevent screenshot override if you have scenarios with the same name in different suites
+-   `keepBrowserState` (optional, default: false)  - keep browser state between tests when `restart` set to false.
 -   `keepCookies` (optional, default: false)  - keep cookies between tests when `restart` set to false.
 -   `waitForAction`: (optional) how long to wait after click, doubleClick or PressKey actions in ms. Default: 500
 -   `waitForTimeout`: (optional) default wait* timeout

docs/helpers/Protractor.md

@@ -25,6 +25,10 @@ This helper should be configured in codecept.json
 -   `driver` - which protrator driver to use (local, direct, session, hosted, sauce, browserstack). By default set to 'hosted' which requires selenium server to be started.
 -   `restart` (optional, default: true) - restart browser between tests.
 -   `smartWait`: (optional) **enables [SmartWait](http://codecept.io/acceptance/#smartwait)**; wait for additional milliseconds for element to appear. Enable for 5 secs: "smartWait": 5000
+-   `disableScreenshots` (optional, default: false)  - don't save screenshot on failure
+-   `uniqueScreenshotNames` (optional, default: false)  - option to prevent screenshot override if you have scenarios with the same name in different suites
+-   `keepBrowserState` (optional, default: false)  - keep browser state between tests when `restart` set to false.
+-   `keepCookies` (optional, default: false)  - keep cookies between tests when `restart` set to false.*
 -   `seleniumAddress` - Selenium address to connect (default: <http://localhost:4444/wd/hub>)
 -   `rootElement` - Root element of AngularJS application (default: body)
 -   `waitForTimeout`: (optional) sets default wait time in _ms_ for all `wait*` functions. 1000 by default.

docs/helpers/SeleniumWebdriver.md

@@ -26,6 +26,9 @@ This helper should be configured in codecept.json
 -   `driver` - which protrator driver to use (local, direct, session, hosted, sauce, browserstack). By default set to 'hosted' which requires selenium server to be started.
 -   `restart` - restart browser between tests (default: true).
 -   `smartWait`: (optional) **enables SmartWait**; wait for additional milliseconds for element to appear. Enable for 5 secs: "smartWait": 5000
+-   `disableScreenshots` (optional, default: false)  - don't save screenshot on failure
+-   `uniqueScreenshotNames` (optional, default: false)  - option to prevent screenshot override if you have scenarios with the same name in different suites
+-   `keepBrowserState` (optional, default: false)  - keep browser state between tests when `restart` set to false.
 -   `keepCookies` (optional, default: false)  - keep cookies between tests when `restart` set to false.*
 -   `seleniumAddress` - Selenium address to connect (default: <http://localhost:4444/wd/hub>)
 -   `waitForTimeout`: (optional) sets default wait time in _ms_ for all `wait*` functions. 1000 by default;

docs/helpers/WebDriverIO.md

@@ -27,6 +27,9 @@ This helper should be configured in codecept.json
 -   `browser` - browser in which perform testing
 -   `restart` (optional, default: true) - restart browser between tests.
 -   `smartWait`: (optional) **enables [SmartWait](http://codecept.io/acceptance/#smartwait)**; wait for additional milliseconds for element to appear. Enable for 5 secs: "smartWait": 5000
+-   `disableScreenshots` (optional, default: false)  - don't save screenshot on failure
+-   `uniqueScreenshotNames` (optional, default: false)  - option to prevent screenshot override if you have scenarios with the same name in different suites
+-   `keepBrowserState` (optional, default: false)  - keep browser state between tests when `restart` set to false.
 -   `keepCookies` (optional, default: false)  - keep cookies between tests when `restart` set to false.
 -   `windowSize`: (optional) default window size. Set to `maximize` or a dimension in the format `640x480`.
 -   `waitForTimeout`: (option) sets default wait time in _ms_ for all `wait*` functions. 1000 by default;

docs/pageobjects.md

@@ -68,6 +68,48 @@ Scenario('login', (I, loginPage) => {
 });
 ```
 
+Also you can use generators inside PageObject:
+```js
+'use strict';
+let I;
+
+module.exports = {
+
+  _init() {
+    I = actor();
+  },
+
+  // setting locators
+  container: "//div[@class = 'numbers']",
+  mainItem: {
+    number: ".//div[contains(@class, 'numbers__main-number')]",
+    title: ".//div[contains(@class, 'numbers__main-title-block')]"
+  },
+
+  // introducing methods
+  openMainArticle: function* () {
+    I.waitForVisible(this.container)
+    let _this = this
+    let title;
+    yield within(this.container, function*(){
+      title = yield I.grabTextFrom(_this.mainItem.number);
+      let subtitle = yield I.grabTextFrom(_this.mainItem.title);
+      title = title + " " + subtitle.charAt(0).toLowerCase() + subtitle.slice(1);
+      yield I.click(_this.mainItem.title)
+    })
+    return title;
+  }
+}
+```
+
+and use them in your tests:
+```js
+Scenario('login2', (I, loginPage, basePage) => {
+  let title = yield* mainPage.openMainArticle()
+  basePage.pageShouldBeOpened(title)
+});
+```
+
 ## Page Fragments
 
 In a similar manner CodeceptJS allows you to generate **PageFragments** and any other are abstraction

docs/reports.md

@@ -141,3 +141,80 @@ codeceptjs run --reporter mochawesome
 
 Result will be located at `output/index.html` file.
 
+### Advanced usage
+Want to have screenshots for failed tests?
+Then add Mochawesome helper to your config:
+```json
+  "helpers": {
+    "Mochawesome": {
+        "uniqueScreenshotNames": "true"
+    }
+  },
+```
+Then tests with failure will have screenshots.
+
+### Configuration
+
+This helper should be configured in codecept.json
+
+-   `uniqueScreenshotNames` (optional, default: false) - option to prevent screenshot override if you have scenarios with the same name in different suites. This option should be the same as in common helper.
+-   `disableScreenshots` (optional, default: false)  - don't save screenshot on failure. This option should be the same as in common helper.
+
+Also if you will add Mochawesome helper, then you will able to add custom context in report:
+#### addMochawesomeContext
+
+Adds context to executed test in HTML report:
+
+```js
+I.addMochawesomeContext('simple string');
+I.addMochawesomeContext('http://www.url.com/pathname');
+I.addMochawesomeContext('http://www.url.com/screenshot-maybe.jpg');
+I.addMochawesomeContext({title: 'expected output',
+                         value: {
+                           a: 1,
+                           b: '2',
+                           c: 'd'
+                         }
+});
+```
+
+**Parameters**
+
+-   `context`  string, url, path to screenshot, object. See [this](https://www.npmjs.com/package/mochawesome#adding-test-context)
+
+## Multi Reports
+Want to use several reporters in the same time? Try to use [mocha-multi](https://www.npmjs.com/package/mocha-multi) reporter
+
+Install it via NPM:
+
+```
+npm i mocha-multi
+```
+
+Configure mocha-multi with reports that you want:
+```json
+  "mocha": {
+    "reporterOptions": {
+      "codeceptjs-cli-reporter": {
+        "stdout": "-",
+        "options": {
+          "verbose": true,
+          "steps": true,
+        }
+      },
+      "mochawesome": {
+       "stdout": "./output/console.log",
+       "options": {
+         "reportDir": "./output",
+         "reportFilename": "report"
+      }
+    }
+  }
+```
+Execute CodeceptJS with mocha-multi reporter:
+
+```
+codeceptjs run --reporter mocha-multi
+```
+
+This will give you cli with steps in console and HTML report in `output` directory

lib/actor.js

@@ -70,6 +70,7 @@ function recordStep(step, args) {
   });
 
   recorder.catchWithoutStop((err) => {
+    step.status = 'failed';
     step.endTime = Date.now();
     event.emit(event.step.failed, step);
     throw err;

lib/codecept.js

@@ -109,7 +109,7 @@ class Codecept {
     if (test) {
       mocha.files = mocha.files.filter((t) => fsPath.basename(t, '_test.js') === test || fsPath.basename(t, '.js') === test || fsPath.basename(t) === test);
     }
-    mocha.run().on('end', () => {
+    mocha.run(() => {
       let done = () => {
         event.emit(event.all.result, this);
       };

lib/helper/Appium.js

@@ -168,25 +168,26 @@ class Appium extends WebdriverIO {
       this.browser = webdriverio.remote(this.options).init();
     }
 
+    let promisesList = [];
     if (this.options.timeouts && this.isWeb) {
-      this.defineTimeout(this.options.timeouts);
+      promisesList.push(this.defineTimeout(this.options.timeouts));
     }
 
     if (this.options.windowSize === 'maximize' && !this.platform) {
-      this.browser.execute('return [screen.width, screen.height]').then((res) => {
+      promisesList.push(this.browser.execute('return [screen.width, screen.height]').then((res) => {
         return this.browser.windowHandleSize({
           width: res.value[0],
           height: res.value[1]
         });
-      });
+      }));
     } else if (this.options.windowSize && this.options.windowSize.indexOf('x') > 0 && !this.platform) {
       let dimensions = this.options.windowSize.split('x');
-      this.browser.windowHandleSize({
+      promisesList.push(this.browser.windowHandleSize({
         width: dimensions[0],
         height: dimensions[1]
-      });
+      }));
     }
-    return this.browser;
+    return this.browser.then(() => Promise.all(promisesList));
   }
 
   _after() {

lib/helper/Mochawesome.js

@@ -0,0 +1,66 @@
+'use strict';
+let addMochawesomeContext;
+let currentTest;
+let currentSuite;
+const Helper = require('../helper');
+const hashCode = require('../utils').hashCode;
+const clearString = require('../utils').clearString;
+const requireg = require('requireg');
+const path = require('path');
+
+
+class Mochawesome extends Helper {
+
+  constructor(config) {
+    super(config);
+    addMochawesomeContext = requireg('mochawesome/addContext');
+    this._createConfig(config);
+  }
+
+  _createConfig(config) {
+
+    // set defaults
+
+    this.options = {
+      uniqueScreenshotNames: false,
+      disableScreenshots: false
+    };
+
+    // override defaults with config
+    Object.assign(this.options, config);
+  }
+
+  _beforeSuite(suite) {
+    currentSuite = suite;
+  }
+
+  _before() {
+    currentTest = {test: currentSuite.ctx.currentTest};
+  }
+
+  _test(test) {
+    currentTest = {test: test};
+  }
+
+
+  _failed(test) {
+    if (this.options.disableScreenshots) return;
+    let fileName = clearString(test.title);
+    if (this.options.uniqueScreenshotNames) {
+      fileName =
+        fileName.substring(0, 10) + '-' + hashCode(test.title) + '-' +
+        hashCode(test.file);
+    }
+    if (test._retries < 1 || test._retries == test.retryNum) {
+      fileName = `${fileName}.failed.png`;
+      return addMochawesomeContext(currentTest, path.join(global.output_dir, fileName));
+    }
+  }
+
+  addMochawesomeContext(context) {
+    return addMochawesomeContext(currentTest, context);
+  }
+
+}
+
+module.exports = Mochawesome;

lib/helper/Nightmare.js

@@ -34,6 +34,9 @@ let withinStatus = false;
  *
  * * `url` - base url of website to be tested
  * * `restart` (optional, default: true) - restart browser between tests.
+ * * `disableScreenshots` (optional, default: false)  - don't save screenshot on failure
+ * * `uniqueScreenshotNames` (optional, default: false)  - option to prevent screenshot override if you have scenarios with the same name in different suites
+ * * `keepBrowserState` (optional, default: false)  - keep browser state between tests when `restart` set to false.
  * * `keepCookies` (optional, default: false)  - keep cookies between tests when `restart` set to false.
  * * `waitForAction`: (optional) how long to wait after click, doubleClick or PressKey actions in ms. Default: 500
  * * `waitForTimeout`: (optional) default wait* timeout
@@ -56,6 +59,7 @@ class Nightmare extends Helper {
       uniqueScreenshotNames: false,
       rootElement: 'body',
       restart: true,
+      keepBrowserState: false,
       keepCookies: false,
       js_errors: null
     };
@@ -196,7 +200,12 @@ class Nightmare extends Helper {
     if (this.options.restart) {
       return this._stopBrowser();
     }
-    if (this.options.keepCookies) return;
+    if (this.options.keepBrowserState) return;
+    if (this.options.keepCookies) {
+      return Promise.all([this.executeScript(function () {
+        return localStorage.clear();
+      })]);
+    }
     return Promise.all([this.browser.cookies.clearAll(), this.executeScript(function () {
       return localStorage.clear();
     })]);